Merge remote-tracking branch 'origin/main' into HEAD

# Conflicts:
#	litellm/litellm_core_utils/streaming_chunk_builder_utils.py
#	tests/unit/litellm_core_utils/test_core_helpers.py

.github/CODEOWNERS

@@ -1,5 +1,6 @@
 /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
+/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /helm/ @yassin-berriai
 /terraform/ @yassin-berriai
 /.github/workflows/zizmor.yml @yuneng-berri

.github/workflows/cost-map-guard.yml

@@ -21,16 +21,18 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.base.sha }}
           persist-credentials: false
-      - name: Fetch the pull request head and its merge base
+      - name: Fetch the pull request head, its merge base, and the base branch tip
         id: revisions
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
-          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+          target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
+          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
           echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
+          echo "target=$target" >> "$GITHUB_OUTPUT"
       - name: Set up uv
         uses: ./.github/actions/setup-uv-with-retries
         with:
@@ -38,6 +40,8 @@ jobs:
       - name: Run the guard
         env:
           MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
+          COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }}
+          COST_MAP_GUARD_AUTHOR_TYPE: ${{ github.event.pull_request.user.type }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           HEAD_REF: ${{ github.event.pull_request.head.ref }}
         run: |

.github/workflows/cost-map-pins-recheck.yml

@@ -0,0 +1,91 @@
+name: Cost map pins recheck
+
+on:
+  push:
+    branches:
+      - main
+      - "litellm_**"
+    paths:
+      - model_prices_and_context_window.json
+      - litellm/model_prices_and_context_window_backup.json
+      - ci_cd/cost_map_pins.json
+
+permissions: {}
+
+jobs:
+  rerun-cost-map-guard:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      actions: write
+      contents: read
+      pull-requests: read
+    steps:
+      - name: Re-run the cost map guard on open pull requests the push could break
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_REF: ${{ github.ref_name }}
+          BEFORE_SHA: ${{ github.event.before }}
+          AFTER_SHA: ${{ github.event.after }}
+          FORCED: ${{ github.event.forced }}
+          COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]'
+          PINS_PATHS: '["ci_cd/cost_map_pins.json"]'
+        run: |
+          if [[ "$BEFORE_SHA" =~ ^0+$ || "$FORCED" == true ]] \
+            || ! gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${AFTER_SHA}?per_page=1" \
+              --jq '(env.PINS_PATHS | fromjson) as $pins | (.files | length) < 300 and all(.files[]; [.filename, .previous_filename] | any(IN($pins[])) | not)' \
+              | grep -qx true; then
+            export RECHECK_PATHS="$COST_MAP_PATHS"
+          else
+            export RECHECK_PATHS="$PINS_PATHS"
+          fi
+          rerun_cutoff="$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ)"
+          candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \
+            -f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) {
+              repository(owner: $owner, name: $name) {
+                pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) {
+                  nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } }
+                  pageInfo { hasNextPage endCursor }
+                }
+              }
+            }' \
+            --jq '(env.RECHECK_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[]
+              | {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)}
+              | select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')"
+          failed=0
+          declare -A rerun_heads
+          while read -r number head_sha listed; do
+            [ -n "$number" ] || continue
+            if [ "$listed" = false ]; then
+              if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \
+                --jq '(env.RECHECK_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then
+                echo "::error::could not list the files of #$number"
+                failed=1
+                continue
+              fi
+              [ -n "$changed" ] || continue
+            fi
+            [ -z "${rerun_heads[$head_sha]:-}" ] || continue
+            rerun_heads[$head_sha]=1
+            if ! runs="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId,createdAt --jq '.[] | "\(.databaseId) \(.createdAt)"')"; then
+              echo "::error::could not list the cost map guard runs of #$number"
+              failed=1
+              continue
+            fi
+            if [ -z "$runs" ]; then
+              echo "::warning::no cost map guard run found for #$number"
+              continue
+            fi
+            while read -r run_id created_at; do
+              if [[ "$created_at" < "$rerun_cutoff" ]]; then
+                echo "::warning::cost map guard run $run_id for #$number is over 30 days old, so GitHub cannot re-run it; #$number gets the pin checks on its next push"
+                continue
+              fi
+              gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true
+              if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then
+                echo "::error::could not re-run cost map guard run $run_id for #$number"
+                failed=1
+              fi
+            done <<< "$runs"
+          done <<< "$candidates"
+          exit "$failed"

.github/workflows/test-linting.yml

@@ -376,10 +376,10 @@ jobs:
             exit 0
           fi
           files="$(git diff --name-only "$base" HEAD)"
-          if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
+          if grep -Eq '^(litellm/|enterprise/|(pyproject\.toml|uv\.lock)$|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
             echo "relevant=true" >> "$GITHUB_OUTPUT"
           else
-            echo "No proxy, types or generator changes in this pull request, nothing to verify."
+            echo "No litellm, enterprise, dependency or generator changes in this pull request, nothing to verify."
             echo "relevant=false" >> "$GITHUB_OUTPUT"
           fi
 

.github/workflows/test-terraform-provider.yml

@@ -12,6 +12,7 @@ on:
     paths:
       - "terraform/provider/**"
       - "litellm/proxy/**"
+      - "enterprise/**"
       - ".github/workflows/test-terraform-provider.yml"
 
 permissions:

.github/workflows/test-unit.yml

@@ -236,6 +236,7 @@ jobs:
               tests/unit/llms/azure_ai
               tests/unit/llms/base_llm
               tests/unit/llms/baseten
+              tests/unit/llms/bigmodel
               tests/unit/llms/black_forest_labs
               tests/unit/llms/brave
               tests/unit/llms/bytez

.grype.yaml

@@ -11,3 +11,12 @@ ignore:
     package:
       name: zlib
       type: apk
+  - vulnerability: GHSA-v4x8-3wr5-3wq8
+    reason: >-
+      filelock ReadWriteLock advisory, and no package in the image takes ReadWriteLock;
+      the fix (4.2.1) is blocked by litellm's own filelock<4.0 cap until #48635 or #48802 lifts it,
+      and the image then moves off 3.32.7, which retires this version-scoped rule
+    package:
+      name: filelock
+      version: 3.32.7
+      type: python

AGENTS.md

@@ -109,6 +109,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - No file sprawl: deliberate file and folder structure
 - Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
 - API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
+- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
 
 Follow conventional commits for commit names and PR titles
 

ci_cd/cost_map_guard.py

@@ -1,37 +1,67 @@
 """Guard the cost map on pull requests.
 
-Every pull request whose diff against its merge base touches one of the two cost map files gets the file
-checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
-skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
-branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
-get the file checks and additionally may only touch those two files and may only add or update models.
+Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the
+file checks: the files parse, the backup copy matches the root file, and every capability pinned in
+ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says
+otherwise) still holds in the tree that merging the pull request into its base branch produces, so a branch cut before
+a pin was added cannot lower it, and a pull request that deletes the pins file fails too. A pull request that leaves
+all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries
+whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named
+litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may
+only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests
+(branches named litellm-providers/price-sync*) and pull requests a bot account opened (COST_MAP_GUARD_AUTHOR_TYPE is
+Bot) get the human checks and may not touch the pins file either.
 """
 
 from __future__ import annotations
 
 import argparse
 import json
+import os
 import subprocess
 import sys
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final
 
 COST_MAP_PATH: Final = "model_prices_and_context_window.json"
 BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
+PINS_PATH: Final = "ci_cd/cost_map_pins.json"
 GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
+CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH)
 BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync"
+TARGET_ENV: Final = "COST_MAP_GUARD_TARGET"
+AUTHOR_TYPE_ENV: Final = "COST_MAP_GUARD_AUTHOR_TYPE"
+BOT_AUTHOR_TYPE: Final = "Bot"
+MERGE_CONFLICT_EXIT_CODE: Final = 1
 SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
 EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
 
 CostMap = dict[str, object]
+_MISSING: Final = object()
 
 
 @dataclass(frozen=True, slots=True)
 class Snapshot:
     cost_map: str
     backup: str
+    pins: str
+
+
+@dataclass(frozen=True, slots=True)
+class MergeFailed:
+    stderr: str
+
+
+@dataclass(frozen=True, slots=True)
+class Pin:
+    key: str
+    field: str
+    value: object
+    checked: str
+    source: str
 
 
 def _parse_object(text: str, path: str) -> CostMap | str:
@@ -91,6 +121,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
     return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
 
 
+def _pin(key: str, field: str, spec: object) -> Pin | str:
+    malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source"
+    if not isinstance(spec, dict) or "value" not in spec:
+        return malformed
+    checked: Final = spec.get("checked")
+    source: Final = spec.get("source")
+    if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source:
+        return malformed
+    return Pin(key, field, spec["value"], checked, source)
+
+
+def _pins(parsed: CostMap) -> Iterator[Pin | str]:
+    for key, fields in parsed.items():
+        if not isinstance(fields, dict):
+            yield f"{PINS_PATH}: {key} must map field names to pins"
+            continue
+        for field, spec in fields.items():
+            yield _pin(key, field, spec)
+
+
+def _pin_violations(parsed: CostMap, entries: Mapping[str, Mapping[str, object]]) -> Iterator[str]:
+    for pin in _pins(parsed):
+        if isinstance(pin, str):
+            yield pin
+        elif pin.key not in entries:
+            yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry"
+        elif entries[pin.key].get(pin.field, _MISSING) != pin.value:
+            yield (
+                f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} "
+                f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first"
+            )
+
+
+def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]:
+    if not pins_text:
+        return (f"{PINS_PATH} is missing; restore it, its pins were verified against live provider calls",)
+    parsed: Final = _parse_object(pins_text, PINS_PATH)
+    if isinstance(parsed, str):
+        return (parsed,)
+    return tuple(_pin_violations(parsed, _entries(head_map)))
+
+
 def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
     base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
     if isinstance(base_map, str):
@@ -120,8 +192,18 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
     )
 
 
+def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]:
+    if PINS_PATH not in changed_files:
+        return ()
+    if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX):
+        return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
+    if author_type == BOT_AUTHOR_TYPE:
+        return (f"PRs opened by a bot may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
+    return ()
+
+
 def touches_cost_map(changed_files: Sequence[str]) -> bool:
-    return any(path in GUARDED_PATHS for path in changed_files)
+    return any(path in CHECKED_PATHS for path in changed_files)
 
 
 def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
@@ -130,9 +212,21 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
     return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
 
 
+def _merged_pin_failures(base: Snapshot, merged: Snapshot | None) -> tuple[str, ...]:
+    if merged is None:
+        return ("the pull request does not merge cleanly into its base branch; merge the base branch in first",)
+    if not merged.pins and not base.pins:
+        return ()
+    merged_map: Final = _parse_object(merged.cost_map, COST_MAP_PATH)
+    if isinstance(merged_map, str):
+        return (f"after merging into the base branch, {merged_map}",)
+    return pin_failures(merged.pins, merged_map)
+
+
 def guard_failures(
     base: Snapshot,
     head: Snapshot,
+    merged: Snapshot | None,
     changed_files: Sequence[str],
     bot: bool,
     *,
@@ -147,6 +241,7 @@ def guard_failures(
         *_file_failures(
             head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
         ),
+        *_merged_pin_failures(base, merged),
         *(_bot_failures(base, head_map, changed_files) if bot else ()),
     )
 
@@ -157,7 +252,22 @@ def _git(*args: str) -> str | None:
 
 
 def snapshot(revision: str) -> Snapshot:
-    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
+    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS))
+
+
+def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None:
+    result: Final = subprocess.run(
+        ("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head),
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    tree: Final = result.stdout.split("\n", 1)[0]
+    if result.returncode == 0:
+        return tree
+    if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree:
+        return None
+    return MergeFailed(result.stderr.strip())
 
 
 def changed_files(base: str, head: str) -> tuple[str, ...] | None:
@@ -169,6 +279,13 @@ def main(argv: Sequence[str]) -> int:
     parser: Final = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--base", required=True, help="merge base of the pull request")
     parser.add_argument("--head", required=True, help="head commit of the pull request")
+    env_target: Final = os.environ.get(TARGET_ENV)
+    parser.add_argument(
+        "--target",
+        default=env_target,
+        required=env_target is None,
+        help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}",
+    )
     parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
     args: Final = parser.parse_args(argv)
     bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
@@ -176,7 +293,19 @@ def main(argv: Sequence[str]) -> int:
     if changed is None:
         print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
         return 1
-    failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
+    if not bot and not touches_cost_map(changed):
+        print(f"cost map guard passed ({contract_for(bot, changed)})")
+        return 0
+    merged: Final = merged_tree(args.base, args.target, args.head)
+    if isinstance(merged, MergeFailed):
+        print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}")
+        return 1
+    failures: Final = (
+        *guard_failures(
+            snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot
+        ),
+        *pin_author_failures(args.head_ref, os.environ.get(AUTHOR_TYPE_ENV, ""), changed),
+    )
     contract: Final = contract_for(bot, changed)
     if failures:
         print(f"cost map guard failed ({contract}):")

ci_cd/cost_map_pins.json

@@ -0,0 +1,16 @@
+{
+    "fireworks_ai/accounts/fireworks/models/minimax-m3": {
+        "supports_vision": {
+            "value": true,
+            "checked": "2026-09-26",
+            "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
+        }
+    },
+    "fireworks_ai/minimax-m3": {
+        "supports_vision": {
+            "value": true,
+            "checked": "2026-09-26",
+            "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
+        }
+    }
+}

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Mapping
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, cast
+from typing import TYPE_CHECKING, Final
 
 from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
     ManagedObjectCreatorAttribution,
@@ -290,12 +290,14 @@ class CheckResponsesCost:
         sees provider env vars, so it fails for every deployment whose credentials
         live in the config; the row then never leaves ``queued``.
         """
-        if not via_router:
-            return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
-        router_response = await self.llm_router.aget_responses(
-            response_id=response_id, litellm_metadata=litellm_metadata
+        retrieved: Final[object] = (
+            await self.llm_router.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
+            if via_router
+            else await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
         )
-        return cast(ResponsesAPIResponse, router_response)
+        if not isinstance(retrieved, ResponsesAPIResponse):
+            raise TypeError(f"Retrieving {response_id} did not return a response object")
+        return retrieved
 
     async def _fetch_unbilled(self, target: _PollTarget) -> ResponsesAPIResponse:
         return await self._get_response(

examples/bigmodel.md

@@ -0,0 +1,63 @@
+# BigModel (Zhipu AI)
+
+Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL
+
+```yaml
+model_list:
+  - model_name: glm
+    litellm_params:
+      model: bigmodel/glm-5.3
+      api_key: os.environ/BIGMODEL_API_KEY
+      api_base: https://open.bigmodel.cn
+```
+
+One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API
+
+| Proxy endpoint | BigModel endpoint |
+| --- | --- |
+| `/v1/chat/completions` | `/api/paas/v4/chat/completions` |
+| `/v1/responses` | `/api/v1/responses` |
+| `/v1/messages` | `/api/anthropic/v1/messages` |
+
+`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix
+
+A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above
+
+The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY`
+
+```python
+import litellm
+
+chat = litellm.completion(
+    model="bigmodel/glm-5.3",
+    messages=[{"role": "user", "content": "Hello"}],
+    thinking={"type": "enabled"},
+    reasoning_effort="low",
+)
+
+response = litellm.responses(
+    model="bigmodel/glm-5.3",
+    input="Hello",
+    reasoning={"effort": "low"},
+    store=True,
+)
+
+async def messages():
+    return await litellm.anthropic.messages.acreate(
+        model="bigmodel/glm-5.3",
+        max_tokens=1024,
+        messages=[{"role": "user", "content": "Hello"}],
+        thinking={"type": "adaptive"},
+        output_config={"effort": "low"},
+    )
+```
+
+Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route
+
+GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters
+
+Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search
+
+The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices
+
+API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27

litellm/__init__.py

@@ -39,6 +39,7 @@ from litellm._logging import (
     turn_on_json,
     log_level,
 )
+from litellm._logging import suppress_startup_output as suppress_startup_output
 import re
 from litellm.constants import (
     DEFAULT_BATCH_SIZE,
@@ -591,7 +592,7 @@ vertex_ai_safety_settings: Optional[dict] = None
 ####### COMPLETION MODELS ###################
 from typing import Set
 
-open_ai_chat_completion_models: Set = set()
+open_ai_chat_completion_models: Set[str] = set()
 open_ai_text_completion_models: Set = set()
 cohere_models: Set = set()
 cohere_chat_models: Set = set()
@@ -619,6 +620,8 @@ vertex_openai_models: Set = set()
 vertex_minimax_models: Set = set()
 vertex_moonshot_models: Set = set()
 vertex_zai_models: Set = set()
+vertex_qwen_models: Final[Set[str]] = set()
+vertex_ai_models: Final[Set[str]] = set()
 ai21_models: Set = set()
 ai21_chat_models: Set = set()
 nlp_cloud_models: Set = set()
@@ -635,6 +638,7 @@ watsonx_models: Set = set()
 gemini_models: Set = set()
 xai_models: Set = set()
 zai_models: Set = set()
+bigmodel_models: Final[set[str]] = set()  # mutable-ok: provider registry is populated and extended at runtime
 deepseek_models: Set = set()
 tencent_models: Set = set()
 runwayml_models: Set = set()
@@ -681,6 +685,7 @@ moonshot_models: Set = set()
 publicai_models: Set = set()
 darkbloom_models: Set = set()
 openinfer_models: Final[Set[str]] = set()
+kyma_models: Final[set[str]] = set()  # mutable-ok: populated when the shared model catalog loads
 streamlake_models: Final[Set[str]] = set()
 zerogpu_models: Final[Set[str]] = set()
 v0_models: Set = set()
@@ -833,6 +838,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
         elif value.get("litellm_provider") == "vertex_ai-zai_models":
             key = key.replace("vertex_ai/", "")
             vertex_zai_models.add(key)
+        elif value.get("litellm_provider") == "vertex_ai-qwen_models":
+            vertex_qwen_models.add(key.removeprefix("vertex_ai/"))
+        elif value.get("litellm_provider") == "vertex_ai":
+            vertex_ai_models.add(key.removeprefix("vertex_ai/"))
         elif value.get("litellm_provider") == "ai21":
             if value.get("mode") == "chat":
                 ai21_chat_models.add(key)
@@ -877,6 +886,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             xai_models.add(key)
         elif value.get("litellm_provider") == "zai":
             zai_models.add(key)
+        elif value.get("litellm_provider") == "bigmodel":
+            bigmodel_models.add(key)
         elif value.get("litellm_provider") == "fal_ai":
             fal_ai_models.add(key)
         elif value.get("litellm_provider") == "deepseek":
@@ -971,6 +982,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             darkbloom_models.add(key)
         elif value.get("litellm_provider") == "openinfer":
             openinfer_models.add(key)
+        elif value.get("litellm_provider") == "kyma":
+            kyma_models.add(key)
         elif value.get("litellm_provider") == "streamlake":
             streamlake_models.add(key)
         elif value.get("litellm_provider") == "zerogpu":
@@ -1107,6 +1120,7 @@ model_list = list(
     | text_completion_inception_models
     | xai_models
     | zai_models
+    | bigmodel_models
     | fal_ai_models
     | deepseek_models
     | modelscope_models
@@ -1148,6 +1162,7 @@ model_list = list(
     | publicai_models
     | darkbloom_models
     | openinfer_models
+    | kyma_models
     | streamlake_models
     | zerogpu_models
     | v0_models
@@ -1204,7 +1219,16 @@ def _build_models_by_provider() -> dict:
         | vertex_deepseek_models
         | vertex_minimax_models
         | vertex_moonshot_models
-        | vertex_zai_models,
+        | vertex_zai_models
+        | vertex_qwen_models
+        | vertex_mistral_models
+        | vertex_llama3_models
+        | vertex_ai_ai21_models
+        | vertex_openai_models
+        | vertex_ai_image_models
+        | vertex_ai_video_models
+        | vertex_embedding_models
+        | vertex_ai_models,
         "ai21": ai21_models | ai21_chat_models,
         "bedrock": bedrock_models | bedrock_converse_models,
         "petals": petals_models,
@@ -1221,6 +1245,7 @@ def _build_models_by_provider() -> dict:
         "text-completion-inception": text_completion_inception_models,
         "xai": xai_models,
         "zai": zai_models,
+        "bigmodel": bigmodel_models,
         "fal_ai": fal_ai_models,
         "deepseek": deepseek_models,
         "tencent": tencent_models,
@@ -1269,6 +1294,7 @@ def _build_models_by_provider() -> dict:
         "publicai": publicai_models,
         "darkbloom": darkbloom_models,
         "openinfer": openinfer_models,
+        "kyma": kyma_models,
         "streamlake": streamlake_models,
         "zerogpu": zerogpu_models,
         "v0": v0_models,

litellm/_internal_context.py

@@ -36,6 +36,8 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families"
 
 _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
 
+_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False)
+
 
 @contextmanager
 def internal_sub_call() -> Generator[None]:
@@ -123,6 +125,30 @@ def current_service_caller() -> str | None:
     return _service_caller.get()
 
 
+@contextmanager
+def mirrored_call() -> Generator[None]:
+    """A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included."""
+    token: Final = _mirrored_call.set(True)
+    try:
+        yield
+    finally:
+        _restore(_mirrored_call, token)
+
+
+def in_mirrored_call() -> bool:
+    return _mirrored_call.get()
+
+
+@contextmanager
+def outside_mirrored_call() -> Generator[None]:
+    """Logging a mirrored call: the calls its callbacks start are not mirrors themselves."""
+    token: Final = _mirrored_call.set(False)
+    try:
+        yield
+    finally:
+        _restore(_mirrored_call, token)
+
+
 @contextmanager
 def pinned_billing_time(moment: datetime) -> Generator[None]:
     """Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read."""

litellm/_logging.py

@@ -10,10 +10,12 @@ import sys
 from collections.abc import Iterator, Mapping
 from datetime import datetime
 from logging import Formatter
+from types import FrameType
 from typing import Final, TextIO
 from urllib.parse import unquote
 
 from pydantic import TypeAdapter
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm.constants import (
@@ -598,6 +600,7 @@ _LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s"
 _LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s"
 _PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX
 _COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}"
+_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m")
 
 
 def _stream_is_tty(stream: TextIO | None) -> bool:
@@ -612,15 +615,28 @@ def _stream_is_tty(stream: TextIO | None) -> bool:
         return False
 
 
-def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
-    """The plain-text log format, colorized only when both streams are an interactive terminal.
+def _color_enabled(stream: TextIO | None) -> bool:
+    """True when ANSI colors written to the stream reach an interactive terminal.
 
     Honors the NO_COLOR convention from no-color.org: color is disabled when
     NO_COLOR is present with a non-empty value.
     """
-    if os.environ.get("NO_COLOR"):
-        return _PLAIN_LOG_FORMAT
-    return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT
+    return not os.environ.get("NO_COLOR") and _stream_is_tty(stream)
+
+
+def _terminal_colors(stdout: TextIO | None, stderr: TextIO | None) -> bool:
+    """True when both streams are an interactive color terminal, the one rule for log lines and startup prints."""
+    return _color_enabled(stdout) and _color_enabled(stderr)
+
+
+def terminal_text(message: str) -> str:
+    """The message with its ANSI colors kept only on a color terminal, by the rule the plain log format uses."""
+    return message if _terminal_colors(sys.stdout, sys.stderr) else _ANSI_ESCAPE.sub("", message)
+
+
+def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
+    """The plain-text log format, colorized only when both streams are an interactive terminal."""
+    return _COLOR_LOG_FORMAT if _terminal_colors(stdout, stderr) else _PLAIN_LOG_FORMAT
 
 
 class LevelRoutingStreamHandler(logging.StreamHandler):
@@ -642,8 +658,8 @@ class LevelRoutingStreamHandler(logging.StreamHandler):
         super().emit(record)
 
 
-def _parse_json_logs_env(value: str | None) -> bool:
-    """Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs.
+def _parse_env_flag(value: str | None) -> bool:
+    """Strict opt-in parse for a boolean env var such as JSON_LOGS: only "true" (any case) enables it.
 
     Matches the reader in litellm-proxy-extras/_logging.py. The previous
     bool(os.getenv(...)) treated any non-empty value, including "false" and "0",
@@ -656,7 +672,18 @@ def resolve_log_level(log_level: str) -> int:
     return getattr(logging, log_level.upper())
 
 
-json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS"))
+json_logs: Final = _parse_env_flag(os.getenv("JSON_LOGS"))
+suppress_startup_output: Final = _parse_env_flag(os.getenv("LITELLM_SUPPRESS_STARTUP_OUTPUT"))
+
+
+class StartupOutputSettings(TypedDict, total=False):
+    json_logs: ReadOnly[object]
+    suppress_startup_output: ReadOnly[object]
+
+
+STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(StartupOutputSettings)
+
+
 # Create a handler for the logger (you may need to adapt this based on your needs)
 log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
 numeric_level: Final[int] = resolve_log_level(log_level)
@@ -743,7 +770,7 @@ class JsonFormatter(Formatter):
         return dt.isoformat()
 
     def format(self, record):
-        message_str: Final = record.getMessage()
+        message_str: Final = _ANSI_ESCAPE.sub("", record.getMessage())
         json_record: Final[dict[str, object]] = {
             "message": message_str,
             "level": record.levelname,
@@ -799,12 +826,14 @@ class CorrelationPlainFormatter(logging.Formatter):
     """Appends trace_id/session_id to plain-text log lines stamped by CorrelationContextFilter.
 
     Mirrors JsonFormatter's handling of these two fields so request_correlation_in_logs
-    behaves the same whether or not json_logs is enabled.
+    behaves the same whether or not json_logs is enabled. Unless it was built with the color format,
+    it also drops the ANSI color codes a message carries, which a piped log would store as text.
     """
 
     def format(self, record: logging.LogRecord) -> str:
         rendered: Final = super().format(record)
-        formatted: Final = rendered if _is_redacted(record) else redact_string(rendered)
+        uncolored: Final = rendered if self._fmt == _COLOR_LOG_FORMAT else _ANSI_ESCAPE.sub("", rendered)
+        formatted: Final = uncolored if _is_redacted(record) else redact_string(uncolored)
         trace_id: Final = getattr(record, "trace_id", None)
         session_id: Final = getattr(record, "session_id", None)
         if not trace_id and not session_id:
@@ -898,29 +927,46 @@ verbose_proxy_logger.addFilter(_diagnostic_filter)
 verbose_proxy_stdout_logger.addFilter(_diagnostic_filter)
 verbose_logger.addFilter(_diagnostic_filter)
 
-_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m")
-
 
 def print_startup_message(message: str) -> None:
     """Write a boot-time message meant for the operator's terminal.
 
-    Plain mode prints it unchanged, colors and spacing included. With JSON logs on, every line of
-    stdout is read by a log pipeline, so the message goes out as one INFO record of the proxy logger
-    instead, on stdout like the print whatever handlers the logging config gave that logger, with the
-    ANSI color codes and surrounding blank lines removed, located at the caller and emitted at any log
-    level, whole and unredacted as the print showed it; a blank spacer produces no record.
+    LITELLM_SUPPRESS_STARTUP_OUTPUT or litellm_settings.suppress_startup_output turns every one of them
+    off; a warning goes through print_startup_warning instead. Plain mode prints it with its spacing,
+    keeping the ANSI colors only on a color terminal by the same check as the plain log format, so a
+    container log never stores escape codes.
+    With JSON logs on, every line of stdout is read by a log pipeline, so the message goes out as one
+    INFO record of the proxy logger instead, on stdout like the print whatever handlers the logging
+    config gave that logger, with the ANSI color codes and surrounding blank lines removed, located at
+    the caller and emitted at any log level, whole and unredacted as the print showed it; a blank
+    spacer produces no record.
     """
+    if litellm.suppress_startup_output is True:
+        return
+    frame: Final = inspect.currentframe()
+    _write_startup_message(message, frame.f_back if frame is not None else None, logging.INFO)
+
+
+def print_startup_warning(message: str) -> None:
+    """Write a boot-time warning as print_startup_message writes a message, even with suppress_startup_output
+    on, since it names something the operator has to fix. With JSON logs on it is a WARNING record, on
+    stdout where the print put it, so a collector that filters on the level field keeps it."""
+    frame: Final = inspect.currentframe()
+    _write_startup_message(message, frame.f_back if frame is not None else None, logging.WARNING)
+
+
+def _write_startup_message(message: str, caller: FrameType | None, level: int) -> None:
     if not litellm.json_logs:
-        print(message)  # noqa: T201  # plain mode: this is the operator-terminal print it replaces
+        import click
+
+        click.echo(terminal_text(message))
         return
     text: Final = _ANSI_ESCAPE.sub("", message).strip("\n")
     if not text:
         return
-    frame: Final = inspect.currentframe()
-    caller: Final = frame.f_back if frame is not None else None
     record: Final = verbose_proxy_logger.makeRecord(
         verbose_proxy_logger.name,
-        logging.INFO,
+        level,
         caller.f_code.co_filename if caller is not None else __file__,
         caller.f_lineno if caller is not None else 0,
         text,
@@ -934,8 +980,8 @@ def print_startup_message(message: str) -> None:
 
 
 def _startup_json_handler() -> logging.Handler:
-    """The proxy's JSON handler on stdout, filters included, for one startup message."""
-    handler: Final = LevelRoutingStreamHandler()
+    """The proxy's JSON handler on stdout, filters included, for one startup message or warning."""
+    handler: Final = logging.StreamHandler(sys.stdout)
     handler.setFormatter(JsonFormatter())
     handler.addFilter(_secret_filter)
     handler.addFilter(_correlation_filter)

litellm/anthropic_beta_gated_fields.py

@@ -0,0 +1,325 @@
+"""Body fields that an ``anthropic-beta`` value gates.
+
+``anthropic_beta_headers_config.json`` says which beta each provider leg
+carries. When a leg drops a beta, the request fields that beta unlocks
+would make the provider 400. Under ``drop_params`` the same request degrades on
+that leg by dropping those fields too; without it the request goes out as sent
+and the provider answers its own 400, like any unsupported param. Every helper
+here is copy-on-write: the caller's containers are returned untouched when
+nothing is stripped, and a shallow copy is returned otherwise, so a fallback
+leg still sends the full request.
+"""
+
+from collections.abc import Mapping, Sequence, Set
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
+from litellm.types.llms.anthropic import (
+    ANTHROPIC_BETA_HEADER_VALUES,
+    ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
+    ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
+    ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,
+    ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
+)
+
+ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21"
+ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15"
+
+THINKING_BLOCK_BINDING: Final = "thinking.block_binding"
+THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates"
+SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at"
+SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config"
+SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal"
+SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)"
+
+_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"})
+
+BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
+    {
+        ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: (THINKING_BLOCK_BINDING,),
+        ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: (THINKING_DISPLAY_UPDATES,),
+        ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: (SYSTEM_CLEAR_AT,),
+        ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: (SYSTEM_OUTPUT_CONFIG,),
+        ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,),
+        ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
+        ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
+    }
+)
+
+
+def _betas_unlocking(field: str) -> frozenset[str]:
+    return frozenset(beta for beta, fields in BETA_GATED_FIELDS.items() if field in fields)
+
+
+_FIELD_BETAS: Final[Mapping[str, frozenset[str]]] = MappingProxyType(
+    {
+        field: _betas_unlocking(field) for fields in BETA_GATED_FIELDS.values() for field in fields
+    }  # comprehension-ok: flattening the beta-to-fields map into a field-to-betas index
+)
+
+
+def field_is_gated(field: str, active_betas: Set[str]) -> bool:
+    """True when no active beta unlocks ``field``, so the field must be stripped."""
+    return _FIELD_BETAS[field].isdisjoint(active_betas)
+
+
+_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+_SEQUENCE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
+_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
+
+
+def _as_mapping(value: object) -> Mapping[str, object] | None:
+    try:
+        return _MAPPING_ADAPTER.validate_python(value, strict=True)
+    except ValidationError:
+        return None
+
+
+def _as_sequence(value: object) -> tuple[object, ...] | None:
+    try:
+        return _SEQUENCE_ADAPTER.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _is_tool_change_block(block: object) -> bool:
+    mapping: Final = _as_mapping(block)
+    block_type: Final = mapping.get("type") if mapping is not None else None
+    return isinstance(block_type, str) and block_type in _TOOL_CHANGE_BLOCK_TYPES
+
+
+def _header_beta_values(headers: Mapping[str, str]) -> frozenset[str]:
+    joined: Final = ",".join(value for key, value in headers.items() if key.lower() == "anthropic-beta")
+    return frozenset(piece.strip() for piece in joined.split(",") if piece.strip())
+
+
+def _validated_headers(headers: object) -> Mapping[str, str]:
+    try:
+        return _HEADERS_ADAPTER.validate_python(headers)
+    except ValidationError:
+        return MappingProxyType({})
+
+
+def active_beta_values(headers: object, body_betas: object = ()) -> frozenset[str]:
+    """The ``anthropic-beta`` values a request will send: the comma-joined header (key matched
+    case-insensitively) plus the body's ``anthropic_beta`` list on legs that carry betas in the body."""
+    body_values: Final = _as_sequence(body_betas) or ()
+    body_betas_as_strings: Final = frozenset(beta for beta in body_values if isinstance(beta, str))
+    return _header_beta_values(_validated_headers(headers)) | body_betas_as_strings
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedThinking:
+    thinking: object
+    stripped: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedMessages:
+    messages: Sequence[object]
+    stripped: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedRequest:
+    request: Mapping[str, object]
+    stripped: tuple[str, ...]
+
+
+def strip_beta_gated_thinking(thinking: object, active_betas: Set[str], *, drop_params: object) -> StrippedThinking:
+    """Drop ``thinking`` sub-fields whose beta is absent. Returns the same object when nothing is gated."""
+    mapping: Final = _as_mapping(thinking)
+    if mapping is None or not drops_unsupported_params(drop_params):
+        return StrippedThinking(thinking=thinking, stripped=())
+    binding_gated: Final = "block_binding" in mapping and field_is_gated(THINKING_BLOCK_BINDING, active_betas)
+    display_gated: Final = mapping.get("display") == "updates" and field_is_gated(
+        THINKING_DISPLAY_UPDATES, active_betas
+    )
+    if not binding_gated and not display_gated:
+        return StrippedThinking(thinking=thinking, stripped=())
+    gated_keys: Final = frozenset(
+        {*(("block_binding",) if binding_gated else ()), *(("display",) if display_gated else ())}
+    )
+    stripped: Final = (
+        *((THINKING_BLOCK_BINDING,) if binding_gated else ()),
+        *((THINKING_DISPLAY_UPDATES,) if display_gated else ()),
+    )
+    return StrippedThinking(
+        thinking={key: value for key, value in mapping.items() if key not in gated_keys},
+        stripped=stripped,
+    )
+
+
+def _strip_beta_gated_system_message(message: Mapping[str, object], active_betas: Set[str]) -> StrippedMessages:
+    """One mid-conversation ``role: system`` message. ``messages`` holds the rewritten message, or nothing
+    when the strip left it with neither content nor ``output_config``, since such a system turn is itself a 400."""
+    clear_at_gated: Final = "clear_at" in message and field_is_gated(SYSTEM_CLEAR_AT, active_betas)
+    output_config_gated: Final = "output_config" in message and field_is_gated(SYSTEM_OUTPUT_CONFIG, active_betas)
+    content: Final = message.get("content")
+    blocks: Final = _as_sequence(content) if not isinstance(content, str) else None
+    tool_changes_gated: Final = (
+        blocks is not None
+        and any(_is_tool_change_block(block) for block in blocks)
+        and field_is_gated(SYSTEM_TOOL_CHANGES, active_betas)
+    )
+    if not clear_at_gated and not output_config_gated and not tool_changes_gated:
+        return StrippedMessages(messages=(message,), stripped=())
+    kept_content: Final[object] = (
+        [block for block in blocks if not _is_tool_change_block(block)]
+        if tool_changes_gated and blocks is not None
+        else content
+    )
+    gated_keys: Final = frozenset(
+        {*(("clear_at",) if clear_at_gated else ()), *(("output_config",) if output_config_gated else ())}
+    )
+    rewritten: Final = {
+        key: (kept_content if key == "content" else value) for key, value in message.items() if key not in gated_keys
+    }
+    stripped: Final = (
+        *((SYSTEM_CLEAR_AT,) if clear_at_gated else ()),
+        *((SYSTEM_OUTPUT_CONFIG,) if output_config_gated else ()),
+        *((SYSTEM_TOOL_CHANGES,) if tool_changes_gated else ()),
+    )
+    if kept_content in (None, "", []) and "output_config" not in rewritten:
+        return StrippedMessages(messages=(), stripped=(*stripped, SYSTEM_MESSAGE_EMPTIED))
+    return StrippedMessages(messages=(rewritten,), stripped=stripped)
+
+
+def _strip_beta_gated_message(message: object, active_betas: Set[str]) -> StrippedMessages:
+    mapping: Final = _as_mapping(message)
+    if mapping is None or mapping.get("role") != "system":
+        return StrippedMessages(messages=(message,), stripped=())
+    return _strip_beta_gated_system_message(mapping, active_betas)
+
+
+def _ends_on_assistant(messages: Sequence[object]) -> bool:
+    last: Final = _as_mapping(messages[-1]) if messages else None
+    return last is not None and last.get("role") == "assistant"
+
+
+def strip_beta_gated_system_messages(
+    messages: Sequence[object], active_betas: Set[str], *, drop_params: object
+) -> StrippedMessages:
+    """Drop the gated fields of every mid-conversation ``role: system`` message. Returns the same
+    sequence when nothing is gated, or when dropping an emptied system turn would leave the conversation
+    ending on an assistant turn, which the model would read as a prefill to continue."""
+    if not drops_unsupported_params(drop_params):
+        return StrippedMessages(messages=messages, stripped=())
+    per_message: Final = tuple(_strip_beta_gated_message(message, active_betas) for message in messages)
+    stripped: Final = tuple(
+        name for result in per_message for name in result.stripped
+    )  # comprehension-ok: flattening per-message strip reports
+    if not stripped:
+        return StrippedMessages(messages=messages, stripped=())
+    kept: Final = [
+        message for result in per_message for message in result.messages
+    ]  # comprehension-ok: flattening per-message results
+    if _ends_on_assistant(kept) and not _ends_on_assistant(messages):
+        return StrippedMessages(messages=messages, stripped=())
+    return StrippedMessages(messages=kept, stripped=stripped)
+
+
+def strip_beta_gated_request_fields(
+    request: Mapping[str, object], active_betas: Set[str], *, drop_params: object
+) -> StrippedRequest:
+    """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
+    nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced."""
+    thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params)
+    messages: Final = request.get("messages")
+    message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
+    system_messages: Final = (
+        strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params)
+        if message_sequence is not None
+        else StrippedMessages(messages=(), stripped=())
+    )
+    stripped: Final = (*thinking.stripped, *system_messages.stripped)
+    if not stripped:
+        return StrippedRequest(request=request, stripped=())
+    rewritten: Final = {
+        **request,
+        **({"thinking": thinking.thinking} if thinking.stripped else {}),
+        **({"messages": system_messages.messages} if system_messages.stripped else {}),
+    }
+    return StrippedRequest(request=rewritten, stripped=stripped)
+
+
+@dataclass(frozen=True, slots=True)
+class StrippedNativeRequest:
+    thinking: object
+    thinking_stripped: bool
+    messages: Sequence[object]
+    messages_stripped: bool
+    stripped: tuple[str, ...]
+
+
+def unstripped_native_request(optional_params: object, messages: object) -> StrippedNativeRequest:
+    """The identity result for legs that strip later, once their final ``anthropic_beta`` list exists."""
+    params: Final = _as_mapping(optional_params) or MappingProxyType({})
+    message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
+    return StrippedNativeRequest(
+        thinking=params.get("thinking"),
+        thinking_stripped=False,
+        messages=message_sequence if message_sequence is not None else (),
+        messages_stripped=False,
+        stripped=(),
+    )
+
+
+def strip_beta_gated_native_request(
+    headers: object, optional_params: object, messages: object, *, body_betas: object = (), drop_params: object
+) -> StrippedNativeRequest:
+    """The ``/v1/messages`` seam, fed the final (already filtered) headers, the final body ``anthropic_beta``
+    list on legs that carry betas in the body, plus the body parts it may rewrite."""
+    if not drops_unsupported_params(drop_params):
+        return unstripped_native_request(optional_params, messages)
+    active_betas: Final = active_beta_values(headers, body_betas)
+    params: Final = _as_mapping(optional_params) or MappingProxyType({})
+    thinking: Final = strip_beta_gated_thinking(params.get("thinking"), active_betas, drop_params=drop_params)
+    message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
+    system_messages: Final = (
+        strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params)
+        if message_sequence is not None
+        else StrippedMessages(messages=(), stripped=())
+    )
+    return StrippedNativeRequest(
+        thinking=thinking.thinking,
+        thinking_stripped=bool(thinking.stripped),
+        messages=system_messages.messages,
+        messages_stripped=bool(system_messages.stripped),
+        stripped=(*thinking.stripped, *system_messages.stripped),
+    )
+
+
+def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider: str) -> None:
+    if not stripped:
+        return
+    described: Final = ", ".join(
+        f"{field} (unlocked by anthropic-beta {' or '.join(sorted(_FIELD_BETAS[field]))})"
+        if field in _FIELD_BETAS
+        else field
+        for field in sorted(set(stripped))
+    )
+    verbose_logger.warning(
+        "Dropped %s for model=%s provider=%s because drop_params is on and the provider would reject them on "
+        "this request, either since this provider leg does not send that anthropic-beta (see "
+        "anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user "
+        "messages.",
+        described,
+        model,
+        provider,
+    )
+
+
+def strip_beta_gated_request_body(
+    request: Mapping[str, object], headers: object, *, drop_params: object, model: str | None, provider: str
+) -> Mapping[str, object]:
+    gated: Final = strip_beta_gated_request_fields(
+        request, active_beta_values(headers, request.get("anthropic_beta") or ()), drop_params=drop_params
+    )
+    log_beta_gated_strips(gated.stripped, model=model, provider=provider)
+    return gated.request

litellm/anthropic_beta_headers_config.json

@@ -248,6 +248,7 @@
     "text_editor_20241022": null,
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
+    "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
     "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": "web-search-2025-03-05"

litellm/anthropic_beta_headers_manager.py

@@ -241,6 +241,13 @@ def get_provider_name(provider: str) -> str:
     return aliases.get(provider, provider)
 
 
+def has_beta_headers_config(provider: str) -> bool:
+    config: Final = _bundled_beta_headers_config()
+    aliases: Final = config.get("provider_aliases", {})
+    canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
+    return canonical_provider in config
+
+
 def filter_and_transform_beta_headers(
     beta_headers: list[str],
     provider: str,
@@ -397,7 +404,7 @@ def update_request_with_filtered_beta(
     headers: dict,
     request_data: dict,
     provider: str,
-) -> tuple[dict, dict]:
+) -> tuple[dict[str, str], dict[str, object]]:
     """
     Update both headers and request body beta fields based on provider support.
     Modifies both dicts in place and returns them.

litellm/caching/caching.py

@@ -455,17 +455,33 @@ class Cache:
         3. Else use the `model` passed in kwargs, plus `model_id` when set (e.g. a SageMaker inference component)
         """
         litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {}
-        metadata_sources: Final[tuple[dict, ...]] = (
-            kwargs.get("metadata") or {},
-            kwargs.get("litellm_metadata") or {},
-            litellm_params.get("metadata") or {},
-            litellm_params.get("litellm_metadata") or {},
+        metadata_sources: Final = cast(  # cast-ok: each source is the untyped request's metadata dict or empty
+            "tuple[dict[str, object], dict[str, object], dict[str, object], dict[str, object]]",
+            (
+                kwargs.get("metadata") or {},
+                kwargs.get("litellm_metadata") or {},
+                litellm_params.get("metadata") or {},
+                litellm_params.get("litellm_metadata") or {},
+            ),
         )
-        model_group: Final[str | None] = next(
-            (source["model_group"] for source in metadata_sources if source.get("model_group")), None
+        metadata, litellm_metadata, params_metadata, params_litellm_metadata = metadata_sources
+        call_model_info: Final[object] = kwargs.get("model_info")
+        routed_source: Final = next(
+            (
+                source
+                for source in (litellm_metadata, metadata, params_litellm_metadata, params_metadata)
+                if call_model_info and source.get("model_group") and source.get("model_info") == call_model_info
+            ),
+            None,
         )
+        model_group: Final = (
+            str(routed_source["model_group"])
+            if routed_source is not None
+            else next((str(source["model_group"]) for source in metadata_sources if source.get("model_group")), None)
+        )
+        caching_group_sources: Final = (routed_source,) if routed_source is not None else metadata_sources
         caching_group: Final = next(
-            (group for source in metadata_sources if (group := self._get_caching_group(source, model_group))), None
+            (group for source in caching_group_sources if (group := self._get_caching_group(source, model_group))), None
         )
         model_id_suffix: Final = (
             f" model_id: {kwargs['model_id']}" if "model_id" in kwargs and kwargs["model_id"] is not None else ""

litellm/caching/qdrant_semantic_cache.py

@@ -13,7 +13,7 @@ import asyncio
 import json
 import os
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Protocol, cast
+from typing import TYPE_CHECKING, Any, Final, Protocol
 
 from pydantic import TypeAdapter
 
@@ -292,10 +292,7 @@ class QdrantSemanticCache(BaseCache):
         prompt: Final = get_semantic_cache_prompt_from_messages(messages)
 
         # create an embedding for prompt
-        embedding_response: Final = cast(
-            EmbeddingResponse,
-            self._get_embedding(prompt, metadata=kwargs.get("metadata")),
-        )
+        embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata"))
 
         # get the embedding
         embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"])
@@ -331,10 +328,7 @@ class QdrantSemanticCache(BaseCache):
         prompt: Final = get_semantic_cache_prompt_from_messages(messages)
 
         # convert to embedding
-        embedding_response: Final = cast(
-            EmbeddingResponse,
-            self._get_embedding(prompt, metadata=kwargs.get("metadata")),
-        )
+        embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata"))
 
         # get the embedding
         embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"])

litellm/caching/redis_semantic_cache.py

@@ -14,7 +14,9 @@ import asyncio
 import json
 import os
 from collections.abc import Callable, Mapping
-from typing import TYPE_CHECKING, Any, Final, cast
+from typing import TYPE_CHECKING, Final, cast
+
+from pydantic import ConfigDict, TypeAdapter
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -38,6 +40,11 @@ if TYPE_CHECKING:
     from litellm.router import Router
 
 
+_CACHE_RESPONSE_ADAPTER: Final[TypeAdapter[str | bytes | None]] = TypeAdapter(
+    str | bytes | None, config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
+
+
 class RedisSemanticCache(BaseCache):
     """
     Redis-backed semantic cache for LLM responses.
@@ -380,7 +387,7 @@ class RedisSemanticCache(BaseCache):
             )
         return embedding_response["data"][0]["embedding"]
 
-    def _get_cache_logic(self, cached_response: Any) -> object:
+    def _get_cache_logic(self, cached_response: str | bytes | None) -> object:
         """
         Process the cached response to prepare it for use.
 
@@ -391,24 +398,23 @@ class RedisSemanticCache(BaseCache):
             The processed cache response, or None if input was None
         """
         if cached_response is None:
-            return cached_response
+            return None
 
         # Convert bytes to string if needed
-        if isinstance(cached_response, bytes):
-            cached_response = cached_response.decode("utf-8")
+        response_text: Final = (
+            cached_response.decode("utf-8") if isinstance(cached_response, bytes) else cached_response
+        )
 
         # Convert string representation to Python object
         try:
-            cached_response = json.loads(cached_response)
+            return json.loads(response_text)
         except json.JSONDecodeError:
             try:
-                cached_response = ast.literal_eval(cached_response)
+                return ast.literal_eval(response_text)
             except (ValueError, SyntaxError) as e:
                 print_verbose(f"Error parsing cached response: {e}")
                 return None
 
-        return cached_response
-
     def set_cache(self, key: str, value: object, **kwargs) -> None:
         """
         Store a value in the semantic cache.
@@ -494,7 +500,7 @@ class RedisSemanticCache(BaseCache):
             similarity: Final = 1 - vector_distance
 
             cached_prompt: Final = cache_hit["prompt"]
-            cached_response: Final = cache_hit["response"]
+            cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"])
 
             # update kwargs["metadata"] with similarity, don't rewrite the original metadata
             kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
@@ -645,7 +651,7 @@ class RedisSemanticCache(BaseCache):
             similarity: Final = 1 - vector_distance
 
             cached_prompt: Final = cache_hit["prompt"]
-            cached_response: Final = cache_hit["response"]
+            cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"])
 
             # update kwargs["metadata"] with similarity, don't rewrite the original metadata
             kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -260,6 +260,7 @@ class ResponsesToCompletionBridgeHandler:
                 structured_output=is_structured_output_request(
                     cast("Mapping[str, object]", optional_params)  # cast-ok: optional_params is an untyped dict
                 ),
+                replays_openai_history=custom_llm_provider == "openai",
             )
             streamwrapper: Final = CustomStreamWrapper(
                 completion_stream=completion_stream,
@@ -363,6 +364,7 @@ class ResponsesToCompletionBridgeHandler:
                 structured_output=is_structured_output_request(
                     cast("Mapping[str, object]", optional_params)  # cast-ok: optional_params is an untyped dict
                 ),
+                replays_openai_history=custom_llm_provider == "openai",
             )
             streamwrapper: Final = CustomStreamWrapper(
                 completion_stream=completion_stream,

litellm/completion_extras/litellm_responses_transformation/history.py

@@ -0,0 +1,432 @@
+import hashlib
+import json
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from itertools import accumulate, chain, pairwise
+from types import MappingProxyType
+from typing import Annotated, Final, Literal, TypeAlias
+
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
+
+REPLAY_FIELD: Final = "openai_responses_replay"
+PHASE_FIELD: Final = "phase"
+
+Phase: TypeAlias = Literal["commentary", "final_answer"]
+
+
+class _Frozen(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+
+class _OutputText(_Frozen):
+    type: Literal["output_text"]
+    text: str
+
+
+class _OutputMessage(_Frozen):
+    type: Literal["message"]
+    role: Literal["assistant"]
+    content: tuple[_OutputText, ...]
+    phase: Phase | None = None
+
+
+class _OutputReasoning(_Frozen):
+    type: Literal["reasoning"]
+    id: str
+
+
+class _OutputToolCall(_Frozen):
+    type: Literal["function_call", "custom_tool_call"]
+    call_id: str
+    id: str | None = None
+
+
+class _OutputToolSearch(_Frozen):
+    type: Literal["tool_search_call", "tool_search_output"]
+
+
+_OutputItem: TypeAlias = Annotated[
+    _OutputMessage | _OutputReasoning | _OutputToolCall | _OutputToolSearch, Field(discriminator="type")
+]
+_OUTPUT_ITEM: Final[TypeAdapter[_OutputItem]] = TypeAdapter(_OutputItem)
+
+
+class _AnyMessage(_Frozen):
+    type: Literal["message"]
+
+
+class _MessageEntry(_Frozen):
+    type: Literal["message"]
+    parts: tuple[Annotated[int, Field(ge=0)], ...]
+    phase: Phase | None = None
+
+
+class _IndexedEntry(_Frozen):
+    type: Literal["reasoning", "function_call", "custom_tool_call", "tool_search_call", "tool_search_output"]
+    index: Annotated[int, Field(ge=0)]
+
+
+_Entry: TypeAlias = Annotated[_MessageEntry | _IndexedEntry, Field(discriminator="type")]
+
+
+class _Replay(_Frozen):
+    version: Literal[1]
+    items: tuple[_Entry, ...]
+    checksum: str
+
+
+class _Namespaced(_Frozen):
+    namespace: str | None = None
+
+
+class _ToolFunction(_Frozen):
+    name: str = ""
+    arguments: str = ""
+    provider_specific_fields: _Namespaced | None = None
+
+
+class _ToolCall(_Frozen):
+    id: str
+    function: _ToolFunction
+    provider_specific_fields: _Namespaced | None = None
+
+
+class _SummaryText(_Frozen):
+    type: str = "summary_text"
+    text: str = ""
+
+
+class _Reasoning(_Frozen):
+    id: str | None = None
+    encrypted_content: str | None = None
+    summary: tuple[_SummaryText, ...] | None = None
+
+
+class _ProviderFields(_Frozen):
+    phase: Phase | None = None
+    tool_search_items: tuple[Mapping[str, object], ...] | None = None
+    openai_responses_replay: object = None
+
+
+class _ChatMessage(_Frozen):
+    role: str | None = None
+    content: str | None = None
+    tool_calls: tuple[_ToolCall, ...] | None = None
+    reasoning_items: tuple[_Reasoning, ...] | None = None
+    phase: Phase | None = None
+    provider_specific_fields: _ProviderFields | None = None
+
+
+class _PhaseFields(_Frozen):
+    phase: Phase | None = None
+
+
+class _RequestedPhaseFields(_Frozen):
+    phase: object = None
+
+
+class _RequestedPhase(_Frozen):
+    phase: object = None
+    provider_specific_fields: _RequestedPhaseFields | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class UnknownPhase:
+    value: object
+
+
+_PHASE: Final[TypeAdapter[Phase]] = TypeAdapter(Phase)
+
+
+class _RetainedFields(_PhaseFields):
+    openai_responses_replay: object = None
+
+
+@dataclass(frozen=True, slots=True)
+class ReplayedTurn:
+    items: tuple[Mapping[str, object], ...]
+    custom_call_ids: frozenset[str]
+
+
+def _as_json(value: object) -> object:
+    return value.model_dump() if isinstance(value, BaseModel) else value
+
+
+def _parsed_output_item(item: object) -> _OutputItem | None:
+    try:
+        return _OUTPUT_ITEM.validate_python(_as_json(item))
+    except ValidationError:
+        return None
+
+
+def _is_message(item: object) -> bool:
+    try:
+        _AnyMessage.model_validate(_as_json(item))
+    except ValidationError:
+        return False
+    return True
+
+
+def _chat_message(message: object) -> _ChatMessage | None:
+    try:
+        return _ChatMessage.model_validate(_as_json(message))
+    except ValidationError:
+        return None
+
+
+def requested_phase(message: object) -> Phase | UnknownPhase | None:
+    try:
+        requested: Final = _RequestedPhase.model_validate(_as_json(message))
+    except ValidationError:
+        return None
+    fields: Final = requested.provider_specific_fields
+    value: Final = requested.phase or (fields.phase if fields else None)
+    if value is None:
+        return None
+    try:
+        return _PHASE.validate_python(value)
+    except ValidationError:
+        return UnknownPhase(value)
+
+
+def reasoning_input_item(
+    item_id: str | None, summary: Sequence[object] | None, encrypted_content: str | None
+) -> Mapping[str, object]:
+    return {
+        "type": "reasoning",
+        **({"id": item_id} if item_id else {}),
+        "summary": list(summary or ()),
+        **({"encrypted_content": encrypted_content} if encrypted_content else {}),
+    }
+
+
+def retained_replay_fields(provider_specific_fields: object) -> Mapping[str, object] | None:
+    try:
+        retained: Final = _RetainedFields.model_validate(provider_specific_fields)
+    except ValidationError:
+        return None
+    return retained.model_dump(exclude_none=True) or None
+
+
+def output_message_text(output_items: Sequence[object], *, first_message_only: bool) -> str:
+    parsed: Final = map(_parsed_output_item, output_items)
+    messages: Final = tuple(item for item in parsed if isinstance(item, _OutputMessage))
+    return "".join(_texts(messages[:1] if first_message_only else messages))
+
+
+def _tool_search_entries(message: _ChatMessage) -> tuple[Mapping[str, object], ...]:
+    fields: Final = message.provider_specific_fields
+    items: Final = (fields.tool_search_items if fields else None) or ()
+    return tuple(item for item in items if item.get("type") in ("tool_search_call", "tool_search_output"))
+
+
+def _checksum(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> str:
+    fields: Final = message.provider_specific_fields
+    bound: Final = {
+        "content": message.content or "",
+        "tool_calls": [[call.id, call.function.name, call.function.arguments] for call in message.tool_calls or ()],
+        "reasoning": [
+            [item.id, item.encrypted_content, [part.text for part in item.summary or ()]]
+            for item in message.reasoning_items or ()
+        ],
+        "tool_search_items": list((fields.tool_search_items if fields else None) or ()),
+        "phase": message.phase or (fields.phase if fields else None),
+        "items": [entry.model_dump(mode="json", exclude_none=True) for entry in entries],
+    }
+    encoded: Final = json.dumps(bound, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return hashlib.sha256(encoded.encode("utf-8", "surrogatepass")).hexdigest()
+
+
+def _kind(item: _OutputItem) -> str:
+    match item:
+        case _OutputToolCall():
+            return "tool_call"
+        case _OutputToolSearch():
+            return "tool_search"
+        case _OutputMessage() | _OutputReasoning():
+            return item.type
+
+
+def _entry(items: Sequence[_OutputItem], position: int) -> _MessageEntry | _IndexedEntry:
+    item: Final = items[position]
+    if isinstance(item, _OutputMessage):
+        return _MessageEntry(type="message", parts=tuple(len(part.text) for part in item.content), phase=item.phase)
+    ordinal: Final = sum(1 for earlier in items[:position] if _kind(earlier) == _kind(item))
+    return _IndexedEntry(type=item.type, index=ordinal)
+
+
+def _texts(items: Sequence[_OutputItem]) -> Iterator[str]:
+    for item in items:
+        if isinstance(item, _OutputMessage):
+            yield from (part.text for part in item.content)
+
+
+def _matches(items: Sequence[_OutputItem], message: _ChatMessage) -> bool:
+    reasoning_ids: Final = tuple(item.id for item in items if isinstance(item, _OutputReasoning))
+    calls: Final = tuple(item for item in items if isinstance(item, _OutputToolCall))
+    tool_calls: Final = message.tool_calls or ()
+    searches: Final = sum(1 for item in items if isinstance(item, _OutputToolSearch))
+    return (
+        "".join(_texts(items)) == (message.content or "")
+        and reasoning_ids == tuple(item.id for item in message.reasoning_items or ())
+        and len(calls) == len(tool_calls)
+        and all(chat.id in (call.call_id, call.id) for call, chat in zip(calls, tool_calls))
+        and searches == len(_tool_search_entries(message))
+    )
+
+
+def _content_phase(
+    output_items: Sequence[object], parsed: Sequence[_OutputItem | None], *, first_message_only: bool
+) -> Phase | None:
+    messages: Final = tuple(item for raw, item in zip(output_items, parsed) if _is_message(raw))
+    if not messages:
+        return None
+    content_message: Final = messages[0] if first_message_only else messages[-1]
+    return content_message.phase if isinstance(content_message, _OutputMessage) else None
+
+
+def _with_phase(message: _ChatMessage, phase: Phase | None) -> _ChatMessage:
+    if phase is None:
+        return message
+    fields: Final = (message.provider_specific_fields or _ProviderFields()).model_copy(update={PHASE_FIELD: phase})
+    return message.model_copy(update={"provider_specific_fields": fields})
+
+
+def replay_fields(
+    output_items: Sequence[object], message: object, *, completed: bool, first_message_only: bool
+) -> Mapping[str, object]:
+    parsed: Final = tuple(map(_parsed_output_item, output_items))
+    phase: Final = _content_phase(output_items, parsed, first_message_only=first_message_only)
+    phase_fields: Final[Mapping[str, object]] = MappingProxyType({PHASE_FIELD: phase} if phase else {})
+    items: Final = tuple(item for item in parsed if item is not None)
+    parsed_message: Final = _chat_message(message)
+    chat: Final = _with_phase(parsed_message, phase) if parsed_message is not None else None
+    if not completed or not items or len(items) != len(parsed) or chat is None or not _matches(items, chat):
+        return phase_fields
+    entries: Final = tuple(_entry(items, position) for position in range(len(items)))
+    replay: Final = _Replay(version=1, items=entries, checksum=_checksum(chat, entries))
+    return MappingProxyType({**phase_fields, REPLAY_FIELD: replay.model_dump(mode="json", exclude_none=True)})
+
+
+def _stored_replay(message: _ChatMessage) -> _Replay | None:
+    fields: Final = message.provider_specific_fields
+    if fields is None or fields.openai_responses_replay is None:
+        return None
+    try:
+        return _Replay.model_validate(fields.openai_responses_replay)
+    except ValidationError:
+        return None
+
+
+def _message_texts(entries: Sequence[_MessageEntry], content: str) -> tuple[tuple[str, ...], ...]:
+    lengths: Final = tuple(chain.from_iterable(entry.parts for entry in entries))
+    bounds: Final = tuple(accumulate(lengths, initial=0))
+    parts: Final = tuple(content[start:end] for start, end in pairwise(bounds))
+    groups: Final = tuple(accumulate((len(entry.parts) for entry in entries), initial=0))
+    return tuple(parts[start:end] for start, end in pairwise(groups))
+
+
+def _message_item(texts: Sequence[str], phase: Phase | None) -> Mapping[str, object]:
+    return {
+        "type": "message",
+        "role": "assistant",
+        "content": [{"type": "output_text", "text": text} for text in texts],
+        **({PHASE_FIELD: phase} if phase else {}),
+    }
+
+
+def _function_call_item(call: _ToolCall) -> Mapping[str, object]:
+    namespaced: Final = call.provider_specific_fields or call.function.provider_specific_fields
+    namespace: Final = namespaced.namespace if namespaced else None
+    return {
+        "type": "function_call",
+        "call_id": call.id,
+        "name": call.function.name,
+        "arguments": call.function.arguments,
+        **({"namespace": namespace} if namespace else {}),
+    }
+
+
+def _custom_tool_call_item(call: _ToolCall) -> Mapping[str, object]:
+    return {
+        "type": "custom_tool_call",
+        "call_id": call.id,
+        "name": call.function.name,
+        "input": call.function.arguments,
+    }
+
+
+def _reasoning_item(item: _Reasoning) -> Mapping[str, object]:
+    summary: Final = tuple(part.model_dump() for part in item.summary or ())
+    return reasoning_input_item(item.id, summary, item.encrypted_content)
+
+
+@dataclass(frozen=True, slots=True)
+class _Resolved:
+    message: _ChatMessage
+    texts: Mapping[int, tuple[str, ...]]
+    tool_searches: tuple[Mapping[str, object], ...]
+
+
+def _input_item(resolved: _Resolved, position: int, entry: _MessageEntry | _IndexedEntry) -> Mapping[str, object]:
+    if isinstance(entry, _MessageEntry):
+        return _message_item(resolved.texts[position], entry.phase)
+    tool_calls: Final = resolved.message.tool_calls or ()
+    match entry.type:
+        case "reasoning":
+            return _reasoning_item((resolved.message.reasoning_items or ())[entry.index])
+        case "function_call":
+            return _function_call_item(tool_calls[entry.index])
+        case "custom_tool_call":
+            return _custom_tool_call_item(tool_calls[entry.index])
+        case "tool_search_call" | "tool_search_output":
+            return dict(resolved.tool_searches[entry.index])
+
+
+def _fits(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> bool:
+    tool_calls: Final = len(message.tool_calls or ())
+    tool_searches: Final = len(_tool_search_entries(message))
+    sizes: Final = MappingProxyType(
+        {
+            "reasoning": len(message.reasoning_items or ()),
+            "function_call": tool_calls,
+            "custom_tool_call": tool_calls,
+            "tool_search_call": tool_searches,
+            "tool_search_output": tool_searches,
+        }
+    )
+    parts: Final = chain.from_iterable(entry.parts for entry in entries if isinstance(entry, _MessageEntry))
+    indexed: Final = (entry for entry in entries if isinstance(entry, _IndexedEntry))
+    return sum(parts) == len(message.content or "") and all(entry.index < sizes[entry.type] for entry in indexed)
+
+
+def _resolve(
+    message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry], replay_tool_search: bool
+) -> ReplayedTurn | None:
+    indexed: Final = tuple(entry for entry in entries if isinstance(entry, _IndexedEntry))
+    if not replay_tool_search and any(entry.type in ("tool_search_call", "tool_search_output") for entry in indexed):
+        return None
+    messages: Final = tuple(entry for entry in entries if isinstance(entry, _MessageEntry))
+    calls: Final = tuple(entry for entry in indexed if entry.type in ("function_call", "custom_tool_call"))
+    texts: Final = _message_texts(messages, message.content or "")
+    positions: Final = tuple(position for position, entry in enumerate(entries) if isinstance(entry, _MessageEntry))
+    resolved: Final = _Resolved(
+        message=message,
+        texts=MappingProxyType(dict(zip(positions, texts, strict=True))),
+        tool_searches=_tool_search_entries(message),
+    )
+    tool_calls: Final = message.tool_calls or ()
+    return ReplayedTurn(
+        items=tuple(_input_item(resolved, position, entry) for position, entry in enumerate(entries)),
+        custom_call_ids=frozenset(tool_calls[entry.index].id for entry in calls if entry.type == "custom_tool_call"),
+    )
+
+
+def replay_items(message: object, *, replay_tool_search: bool) -> ReplayedTurn | None:
+    chat: Final = _chat_message(message)
+    stored: Final = _stored_replay(chat) if chat is not None and chat.role == "assistant" else None
+    if chat is None or stored is None or stored.checksum != _checksum(chat, stored.items):
+        return None
+    if not _fits(chat, stored.items):
+        return None
+    return _resolve(chat, stored.items, replay_tool_search)

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -24,9 +24,19 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 import litellm
 from litellm import ModelResponse
 from litellm._logging import verbose_logger
+from litellm.completion_extras.litellm_responses_transformation.history import (
+    PHASE_FIELD,
+    UnknownPhase,
+    output_message_text,
+    reasoning_input_item,
+    replay_fields,
+    replay_items,
+    requested_phase,
+)
 from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
 from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    prompt_cache_breakpoint_for_wire,
     responses_reasoning_items_from_thinking_blocks,
     with_prompt_cache_breakpoint,
 )
@@ -52,7 +62,6 @@ from litellm.types.llms.openai import (
     ChatCompletionToolCallChunk,
     ChatCompletionToolCallFunctionChunk,
     ChatCompletionToolParamFunctionChunk,
-    PromptCacheBreakpoint,
     Reasoning,
     ResponsesAPIOptionalRequestParams,
     ResponsesAPIResponse,
@@ -490,7 +499,7 @@ def _fold_choices_into_one(
     choices: Sequence["Choices"],
     tool_calls: Sequence[Mapping[str, object]],
     trailing_reasoning_content: str | None,
-    trailing_reasoning_item: _BuiltReasoningItem | None,
+    trailing_reasoning_items: Sequence[_BuiltReasoningItem],
     tool_search_items: Sequence[Mapping[str, object]],
     structured_output: bool = False,
 ) -> "Choices":
@@ -509,7 +518,7 @@ def _fold_choices_into_one(
     reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
     reasoning_items: Final = (
         *chain.from_iterable(_message_reasoning_items(m) for m in messages),
-        *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
+        *trailing_reasoning_items,
     )
     return Choices(
         message=Message(
@@ -526,6 +535,61 @@ def _fold_choices_into_one(
     )
 
 
+def _with_replay_fields(
+    choice: "Choices", output_items: Sequence[object], *, completed: bool, structured_output: bool
+) -> "Choices":
+    fields: Final = {
+        **(choice.message.provider_specific_fields or {}),
+        **replay_fields(output_items, choice.message, completed=completed, first_message_only=structured_output),
+    }
+    message: Final = choice.message.model_copy(update={"provider_specific_fields": fields or None})
+    return choice.model_copy(update={"message": message})
+
+
+_STREAMED_OUTPUT_ITEMS: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
+    tuple[Mapping[str, object], ...]
+)
+
+
+def completed_stream_history(
+    output_payload: object,
+    tool_search_fields: Mapping[str, object] | None,
+    *,
+    completed: bool,
+    structured_output: bool,
+) -> dict[str, object] | None:  # mutable-ok: Delta.provider_specific_fields is typed as a dict
+    items: Final = _STREAMED_OUTPUT_ITEMS.validate_python(output_payload)
+    calls: Final = tuple(item for item in items if item.get("type") in ("function_call", "custom_tool_call"))
+    streamed_message: Final = {
+        "role": "assistant",
+        "content": output_message_text(items, first_message_only=structured_output),
+        "tool_calls": [tool_call_dict_from_output_item(call, index) for index, call in enumerate(calls)],
+        "reasoning_items": list(_reasoning_items_from_output_items(items)),
+        "provider_specific_fields": tool_search_fields,
+    }
+    fields: Final = {
+        **(tool_search_fields or {}),
+        **replay_fields(items, streamed_message, completed=completed, first_message_only=structured_output),
+    }
+    return fields or None
+
+
+def _assistant_phase_field(msg: object, replay_openai_history: bool, drop_params: bool) -> Mapping[str, object]:
+    match requested_phase(msg) if replay_openai_history else None:
+        case None:
+            return {}
+        case UnknownPhase(value=value) if drop_params:
+            verbose_logger.debug("Chat provider: dropping assistant phase=%r under drop_params", value)
+            return {}
+        case UnknownPhase(value=value):
+            raise litellm.UnsupportedParamsError(
+                message=f"phase must be 'commentary' or 'final_answer', got {value!r}. "
+                "To drop it, set `litellm.drop_params=True` or pass `drop_params=True`."
+            )
+        case phase:
+            return {PHASE_FIELD: phase}
+
+
 def _responses_event_type(chunk: Mapping[str, object]) -> object:
     event_type: Final = chunk.get("type")
     return event_type.value if isinstance(event_type, ResponsesAPIStreamEvents) else event_type
@@ -567,19 +631,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li
     return "length"
 
 
-_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
-
-
-def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
-    if marker is None or not drop_params:
-        return marker
-    try:
-        return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
-    except ValidationError:
-        verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
-        return None
-
-
 def _input_file_from_file_value(file_value: object) -> dict[str, object]:
     if not isinstance(file_value, dict):
         return {"type": "input_file"}
@@ -667,15 +718,7 @@ def _reasoning_item_to_response_input(
     An item without an id is sent without one: the Responses API accepts that and
     verifies the encrypted content on its own, while it rejects any id it did not mint.
     """
-    item_id: Final = r_item.get("id")
-    encrypted_content: Final = r_item.get("encrypted_content")
-    return {
-        "type": "reasoning",
-        **({"id": item_id} if item_id else {}),
-        # summary is always required by the Responses API, even when empty
-        "summary": r_item.get("summary") or [],
-        **({"encrypted_content": encrypted_content} if encrypted_content else {}),
-    }
+    return dict(reasoning_input_item(r_item.get("id"), r_item.get("summary"), r_item.get("encrypted_content")))
 
 
 def _requested_defer_loading(tool: Mapping[str, object], function_tool: Mapping[str, object]) -> object:
@@ -804,12 +847,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         keep_prompt_cache_breakpoints: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
+        replay_openai_history: bool = False,
     ) -> tuple[list[object], str | None]:
         converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
             messages,
             drop_params=drop_params,
             replay_tool_search=replay_tool_search,
             include_namespace=include_namespace,
+            replay_openai_history=replay_openai_history,
         )
         return (
             converted_input_items
@@ -825,10 +870,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         drop_params: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
+        replay_openai_history: bool = False,
     ) -> tuple[list[object], str | None]:
         input_items: Final[list[object]] = []
         instructions: str | None = None
-        custom_tool_call_ids: Final = frozenset(
+        replayed_turns: Final = tuple(
+            replay_items(msg, replay_tool_search=replay_tool_search) if replay_openai_history else None
+            for msg in messages
+        )
+        replayed_custom_call_ids: Final = frozenset(
+            chain.from_iterable(turn.custom_call_ids for turn in replayed_turns if turn is not None)
+        )
+        custom_tool_call_ids: Final = replayed_custom_call_ids | frozenset(
             tool_call["id"]
             for msg in messages
             if msg.get("role") == "assistant" and isinstance(msg.get("tool_calls"), list)
@@ -849,7 +902,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             tool_calls = msg.get("tool_calls")
             tool_call_id = msg.get("tool_call_id")
 
-            if role == "system":
+            if (replayed_turn := replayed_turns[index]) is not None:
+                input_items.extend(replayed_turn.items)
+            elif role == "system":
                 # Extract system message as instructions
                 if isinstance(content, str) and index < leading_system_count:
                     if instructions:
@@ -915,6 +970,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             "content": self._convert_content_to_responses_format(
                                 content, "assistant", drop_params=drop_params
                             ),
+                            **_assistant_phase_field(msg, replay_openai_history, drop_params),
                         }
                     )
                 for tool_call in tool_calls:
@@ -955,6 +1011,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         "content": self._convert_content_to_responses_format(
                             content, cast(str, role), drop_params=drop_params
                         ),
+                        **_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
                     }
                 )
             elif role == "assistant":
@@ -1081,6 +1138,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
             replay_tool_search=_requests_tool_search(cast(object, optional_params)),  # cast-ok: upcast
             include_namespace=True,
+            replay_openai_history=provider == "openai",
         )
         # OpenAI's Responses API rejects an empty input. For a system-only
         # request, carry the system message as a system-role input item instead
@@ -1165,6 +1223,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         output_items: Sequence[object],
         handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None,
         structured_output: bool = False,
+        replay_openai_history: bool = False,
+        completed: bool = False,
     ) -> list["Choices"]:
         """
         Convert Responses API output items to chat completion choices.
@@ -1174,6 +1234,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             handle_raw_dict_callback: Optional callback for handling raw dict items
             structured_output: The chat request carried a json response_format, so only the
                 first message item becomes content
+            replay_openai_history: Record the output's item sequence for the next turn
+            completed: The response finished, so its item sequence is whole
 
         Returns:
             List of Choices objects
@@ -1196,7 +1258,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         choices: Final[list[Choices]] = []
         index = 0
         reasoning_content: str | None = None
-        pending_reasoning_item: _BuiltReasoningItem | None = None
+        pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = ()
 
         # Collect all tool calls to put them in a single choice
         # (Chat Completions API expects all tool calls in one message)
@@ -1205,12 +1267,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         for item in output_items:
             if isinstance(item, ResponseReasoningItem):
-                pending_reasoning_item = _build_reasoning_item(
-                    item_id=item.id,
-                    encrypted_content=getattr(item, "encrypted_content", None),
-                    summary_raw=item.summary,
+                pending_reasoning_items = (
+                    *(pending_reasoning_items if replay_openai_history else ()),
+                    _build_reasoning_item(
+                        item_id=item.id,
+                        encrypted_content=getattr(item, "encrypted_content", None),
+                        summary_raw=item.summary,
+                    ),
+                )
+                reasoning_content = " ".join(
+                    s["text"]
+                    for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items)
+                    if s.get("text")
                 )
-                reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text"))
 
             elif isinstance(item, ResponseOutputMessage):
                 for content in item.content:
@@ -1227,7 +1296,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         annotations=annotations,
                         reasoning_items=cast(
                             list[ChatCompletionReasoningItem] | None,
-                            ([pending_reasoning_item] if pending_reasoning_item is not None else None),
+                            (list(pending_reasoning_items) or None),
                         ),
                     )
 
@@ -1240,7 +1309,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     )
 
                     reasoning_content = None  # flush
-                    pending_reasoning_item = None  # flush
+                    pending_reasoning_items = ()  # flush
                     index += 1
 
             elif isinstance(item, ResponseFunctionToolCall):
@@ -1300,16 +1369,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     output_items, "stop", tool_search_items=tool_search_items
                 )
             ]
-        return [
-            _fold_choices_into_one(
-                choices=choices,
-                tool_calls=accumulated_tool_calls,
-                trailing_reasoning_content=reasoning_content,
-                trailing_reasoning_item=pending_reasoning_item,
-                tool_search_items=tool_search_items,
-                structured_output=structured_output,
-            )
-        ]
+        folded: Final = _fold_choices_into_one(
+            choices=choices,
+            tool_calls=accumulated_tool_calls,
+            trailing_reasoning_content=reasoning_content,
+            trailing_reasoning_items=pending_reasoning_items,
+            tool_search_items=tool_search_items,
+            structured_output=structured_output,
+        )
+        if not replay_openai_history:
+            return [folded]
+        return [_with_replay_fields(folded, output_items, completed=completed, structured_output=structured_output)]
 
     @staticmethod
     def _build_empty_choice(
@@ -1443,6 +1513,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             structured_output=is_structured_output_request(
                 cast("Mapping[str, object]", optional_params)  # cast-ok: optional_params is an untyped dict
             ),
+            replay_openai_history=_served_provider(
+                cast("Mapping[str, object]", litellm_params)  # cast-ok: litellm_params is an untyped dict
+            )
+            == "openai",
+            completed=raw_response.status == "completed",
         )
 
         response_is_incomplete: Final = raw_response.status == "incomplete" or (
@@ -1506,9 +1581,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         sync_stream: bool,
         json_mode: bool | None = False,
         structured_output: bool = False,
+        replays_openai_history: bool = False,
     ) -> BaseModelResponseIterator:
         return OpenAiResponsesToChatCompletionStreamIterator(
-            streaming_response, sync_stream, json_mode, structured_output=structured_output
+            streaming_response,
+            sync_stream,
+            json_mode,
+            structured_output=structured_output,
+            replays_openai_history=replays_openai_history,
         )
 
     def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
@@ -1586,7 +1666,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     if original_type == "text":
                         converted = with_prompt_cache_breakpoint(
                             self._convert_content_str_to_input_text(item.get("text", ""), role),
-                            _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   text -> %s", converted)
@@ -1599,14 +1679,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                     cast(ChatCompletionImageObject, item), role
                                 ),
                             ),
-                            _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   image_url -> %s", converted)
                     elif original_type == "input_audio":
                         converted = with_prompt_cache_breakpoint(
                             {"type": "input_audio", "input_audio": item.get("input_audio")},
-                            _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   input_audio -> %s", converted)
@@ -1622,7 +1702,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                 _input_file_from_file_value(
                                     cast("ChatCompletionFileObject", item).get("file"),  # cast-ok: type tag checked
                                 ),
-                                _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   file -> %s", converted)
@@ -1646,7 +1726,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             # Default to input_text for unknown types
                             converted = with_prompt_cache_breakpoint(
                                 self._convert_content_str_to_input_text(str(item.get("text", item)), role),
-                                _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   unknown(%s) -> %s", original_type, converted)
@@ -1890,8 +1970,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         sync_stream: bool,
         json_mode: bool | None = False,
         structured_output: bool = False,
+        replays_openai_history: bool = False,
     ):
         super().__init__(streaming_response, sync_stream, json_mode)
+        self._replays_openai_history: Final = replays_openai_history
         self._chat_completion_id: str | None = None
         self._served_service_tier: str | None = None
         self._tool_call_index_map: dict[int, int] = {}  # mutable-ok: per-stream accumulator state
@@ -1937,6 +2019,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
     def translate_responses_chunk_to_openai_stream(
         parsed_chunk: dict | BaseModel,
         tool_call_index_map: dict[int, int] | None = None,  # mutable-ok: per-stream state, remapped in place
+        replays_openai_history: bool = False,
+        structured_output: bool = False,
     ) -> "ModelResponseStream":
         """
         Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
@@ -1944,6 +2028,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         Args:
             parsed_chunk: Dict containing the Responses API event chunk
             tool_call_index_map: Per-stream output_index -> sequential tool_call index map
+            replays_openai_history: Record the completed output's item sequence for the next turn
+            structured_output: Only the first message item became content
 
         Returns:
             ModelResponseStream: OpenAI-formatted streaming chunk
@@ -2154,6 +2240,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             terminal_reasoning_items_typed: Final = _as_chat_reasoning_items(
                 _reasoning_items_from_output_items(output_items)
             )
+            output_payload: Final = cast(object, output_items)  # cast-ok: untyped
+            tool_search_fields: Final = _tool_search_provider_fields(
+                _tool_search_items_from_output_items(output_payload)
+            )
 
             usage = None
             if response_data.get("usage"):
@@ -2167,10 +2257,17 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                     StreamingChoices(
                         index=0,
                         delta=Delta(
-                            content="",
+                            content=None if replays_openai_history else "",
                             reasoning_items=terminal_reasoning_items_typed,
-                            provider_specific_fields=_tool_search_provider_fields(
-                                _tool_search_items_from_output_items(cast(object, output_items))  # cast-ok: untyped
+                            provider_specific_fields=(
+                                completed_stream_history(
+                                    output_payload,
+                                    tool_search_fields,
+                                    completed=event_type == "response.completed",
+                                    structured_output=structured_output,
+                                )
+                                if replays_openai_history
+                                else tool_search_fields
                             ),
                         ),
                         finish_reason=finish_reason,
@@ -2216,7 +2313,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             self._empty_content_chunk()
             if self._is_text_delta_after_structured_answer(chunk)
             else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
-                chunk, tool_call_index_map=self._tool_call_index_map
+                chunk,
+                tool_call_index_map=self._tool_call_index_map,
+                replays_openai_history=self._replays_openai_history,
+                structured_output=self._structured_output,
             )
         )
         if self._structured_output and _is_message_item_done(chunk):

litellm/constants.py

@@ -158,7 +158,7 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_
 DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250)
 HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
 HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range(
-    "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400
+    "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 60, 1, 86400
 )
 
 # Maximum wall-clock seconds a streaming response is allowed to run.
@@ -627,6 +627,7 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
 # Anthropic's Messages API rejects thinking.budget_tokens < 1024.
 ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
+ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
 REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
 DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
 DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096

litellm/cost_calculator.py

@@ -22,7 +22,7 @@ from litellm.constants import (
     OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
     REALTIME_METERED_USAGE_KEY,
 )
-from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
 from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
 from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
@@ -50,6 +50,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
     parse_prompt_tokens_details,
     select_cost_metric_for_model,
 )
+from litellm.litellm_core_utils.llm_response_utils.response_metadata import response_ms_since_logging_start
 from litellm.llms.anthropic.cost_calculation import (
     cost_per_token as anthropic_cost_per_token,
 )
@@ -1396,9 +1397,9 @@ def _apply_cost_margin(
 
 def _store_cost_breakdown_in_logging_obj(
     litellm_logging_obj: LitellmLoggingObject | None,
-    prompt_tokens_cost_usd_dollar: float,
-    completion_tokens_cost_usd_dollar: float,
-    cost_for_built_in_tools_cost_usd_dollar: float,
+    prompt_tokens_cost_usd_dollar: float | None,
+    completion_tokens_cost_usd_dollar: float | None,
+    cost_for_built_in_tools_cost_usd_dollar: float | None,
     total_cost_usd_dollar: float,
     additional_costs: dict | None = None,
     original_cost: float | None = None,
@@ -1500,9 +1501,45 @@ def _response_time_ms_for_cost(
         return float(stamped)
     if total_time:
         return total_time
-    if litellm_logging_obj is not None:
-        return litellm_logging_obj.get_response_ms()
-    return 0.0
+    if litellm_logging_obj is None:
+        return 0.0
+    reported_ms: Final = response_ms_since_logging_start(litellm_logging_obj)
+    if reported_ms is not None and litellm_logging_obj.stream is not True:
+        return reported_ms
+    return litellm_logging_obj.get_response_ms()
+
+
+def _adjusted_provider_billed_cost(
+    reported_cost: float,
+    custom_llm_provider: str | None,
+    model: str | None,
+    region_name: str | None,
+    litellm_logging_obj: LitellmLoggingObject | None,
+    service_tier: str | None,
+    data_residency: str | None,
+) -> float:
+    discounted_cost, discount_percent, discount_amount = _apply_cost_discount(
+        base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name
+    )
+    adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin(
+        base_cost=discounted_cost, custom_llm_provider=custom_llm_provider
+    )
+    _store_cost_breakdown_in_logging_obj(
+        litellm_logging_obj=litellm_logging_obj,
+        prompt_tokens_cost_usd_dollar=None,
+        completion_tokens_cost_usd_dollar=None,
+        cost_for_built_in_tools_cost_usd_dollar=None,
+        total_cost_usd_dollar=adjusted_cost,
+        original_cost=reported_cost,
+        discount_percent=discount_percent,
+        discount_amount=discount_amount,
+        margin_percent=margin_percent,
+        margin_fixed_amount=margin_fixed_amount,
+        margin_total_amount=margin_total_amount,
+        service_tier=service_tier,
+        data_residency=data_residency,
+    )
+    return adjusted_cost
 
 
 def completion_cost(
@@ -1616,6 +1653,9 @@ def completion_cost(
             )
 
         explicit_pricing: Final = custom_pricing is True or base_model is not None
+        caller_set_prices: Final = (
+            explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None
+        )
         custom_priced_request_model: Final = model if custom_pricing is True else None
         configured_cost_per_query: Final = (
             _configured_search_cost_per_query(litellm_logging_obj) if custom_pricing else None
@@ -1666,6 +1706,7 @@ def completion_cost(
             raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
         )
 
+        priced_response: Final[object] = completion_response
         for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names):
             model = candidate_model
             try:
@@ -1984,6 +2025,19 @@ def completion_cost(
                     total_time=total_time,
                 )
                 model_for_discount = custom_priced_request_model or model
+                if (
+                    not caller_set_prices
+                    and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None
+                ):
+                    return _adjusted_provider_billed_cost(
+                        reported_cost=reported_cost,
+                        custom_llm_provider=custom_llm_provider,
+                        model=model_for_discount,
+                        region_name=region_name if isinstance(region_name, str) else None,
+                        litellm_logging_obj=litellm_logging_obj,
+                        service_tier=service_tier,
+                        data_residency=data_residency,
+                    )
                 # Calculate cost based on prompt_tokens, completion_tokens
                 if (
                     "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"

litellm/images/main.py

@@ -641,6 +641,8 @@ def image_generation(
                     timeout=timeout,
                     client=custom_client,
                 )
+        elif custom_llm_provider == LlmProviders.KYMA.value:
+            raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
 
         return model_response
     except Exception as e:

litellm/integrations/anthropic_cache_control_hook.py

@@ -72,6 +72,8 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
 OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
 _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
+_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
+_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
 OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
     {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"}
 )
@@ -125,6 +127,10 @@ def _served_provider_for_model(model: str) -> str | None:
 
 
 def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
+    return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True
+
+
+def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None:
     hosted_flag: Final = (
         None
         if custom_llm_provider is None
@@ -135,9 +141,19 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str
     model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
     if model_map_flag is not None:
         return model_map_flag
-    version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower())
-    if version_match is None:
+    fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1])
+    if fine_tuned_base is not None:
+        return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None
+    return _named_model_breakpoint_support(model)
+
+
+def _named_model_breakpoint_support(model: str) -> bool | None:
+    bare_model: Final = model.rsplit("/", 1)[-1].lower()
+    if _O_SERIES_PATTERN.match(bare_model) is not None:
         return False
+    version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
+    if version_match is None:
+        return None
     version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
     return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
 

litellm/integrations/custom_logger.py

@@ -513,7 +513,7 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         data: dict,
         user_api_key_dict: UserAPIKeyAuth,
         call_type: CallTypesLiteral,
-    ) -> Any:
+    ) -> object:
         pass
 
     async def async_post_call_streaming_hook(

litellm/integrations/datadog/datadog_cost_management.py

@@ -205,11 +205,12 @@ class DatadogCostManagementLogger(CustomBatchLogger):
         # see _RESERVED_TAG_KEYS for the rationale.
         if self.cost_tag_keys:
             allow: Final = set(self.cost_tag_keys)
+            client_allow: Final = allow - {"is_silent_experiment"}
             for rt in log.get("request_tags") or []:
                 if not isinstance(rt, str) or ":" not in rt:
                     continue
                 k, _, v = rt.partition(":")
-                if k in allow and v:
+                if k in client_allow and v:
                     self._set_custom_tag(tags, k, v)
             for k, v in metadata.items():
                 if k in allow and v is not None and not isinstance(v, (dict, list)):
@@ -218,7 +219,7 @@ class DatadogCostManagementLogger(CustomBatchLogger):
                 nested = metadata.get(nested_key)
                 if isinstance(nested, dict):
                     for k, v in nested.items():
-                        if k in allow and v is not None and not isinstance(v, (dict, list)):
+                        if k in client_allow and v is not None and not isinstance(v, (dict, list)):
                             self._set_custom_tag(tags, k, str(v))
 
         return tags

litellm/integrations/langfuse/langfuse.py

@@ -833,6 +833,12 @@ class LangFuseLogger:
                 ("vertex_location", vertex_location, bool(vertex_location)),
                 ("aws_region_name", aws_region_name, bool(aws_region_name)),
                 ("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs),
+                (
+                    "is_silent_experiment",
+                    "is_silent_experiment" in allowlisted_metadata
+                    and allowlisted_metadata["is_silent_experiment"] is True,
+                    "is_silent_experiment" in allowlisted_metadata,
+                ),
             )
             enrichments: Final[Mapping[str, object]] = {
                 key: value for key, value, include in candidate_enrichments if include

litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py

@@ -15,7 +15,7 @@ class BaseLLMObsOTELAttributes(ABC):
         pass
 
 
-def cast_as_primitive_value_type(value) -> str | bool | int | float:
+def cast_as_primitive_value_type(value: object) -> str | bool | int | float:
     """
     Converts a value to an OTEL-supported primitive for Arize/Phoenix observability.
     """

litellm/integrations/prometheus.py

@@ -1173,8 +1173,8 @@ class PrometheusLogger(CustomLogger):
         # Build label filters from valid configurations
         label_filters: Final = self._build_label_filters(parsed_configs)
 
-        # Pretty print the processed configuration
-        self._pretty_print_prometheus_config(label_filters)
+        if litellm.suppress_startup_output is not True:
+            self._pretty_print_prometheus_config(label_filters)
         return label_filters
 
     def _parse_exclude_config(self) -> tuple[frozenset[str], frozenset[str]]:
@@ -5148,6 +5148,12 @@ def _get_combined_custom_metadata_from_standard_logging_payload(
         **(requester_metadata if isinstance(requester_metadata, dict) else {}),
         **(user_api_key_auth_metadata if isinstance(user_api_key_auth_metadata, dict) else {}),
         **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}),
+        "is_silent_experiment": (
+            "true"
+            if "is_silent_experiment" in standard_logging_metadata
+            and standard_logging_metadata["is_silent_experiment"] is True
+            else None
+        ),
     }
 
 

litellm/integrations/websearch_interception/handler.py

@@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import (
 from litellm.litellm_core_utils.agentic_loop_settings import (
     validated_max_agentic_loops,
 )
+from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model
 from litellm.llms.base_llm.search.transformation import SearchResponse
 from litellm.types.integrations.custom_logger import (
     CHAT_COMPLETION_AGENTIC_SURFACE,
@@ -114,6 +115,18 @@ class _WebSearchSettingsView(TypedDict):
     websearch_interception_params: WebSearchInterceptionConfig
 
 
+def _caller_model(model: str, logging_obj: "LiteLLMLoggingObj | None", kwargs: Mapping[str, object]) -> str:
+    agentic_view: Final[_AgenticLoopParamsView] = {
+        "agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {}) if logging_obj else {}
+    }
+    recorded_model: Final = agentic_view["agentic_loop_params"].get("model")
+    custom_llm_provider: Final = kwargs.get("custom_llm_provider")
+    return qualify_provider_stripped_model(
+        recorded_model if isinstance(recorded_model, str) and recorded_model else model,
+        custom_llm_provider if isinstance(custom_llm_provider, str) else "",
+    )
+
+
 class _SearchToolLitellmParams(TypedDict, total=False):
     search_provider: ReadOnly[str | None]
 
@@ -1155,6 +1168,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             tool_calls=tool_calls,
             optional_params=optional_params,
             kwargs=kwargs,
+            logging_obj=logging_obj,
             response_format=response_format,
         )
         return AgenticLoopPlan(
@@ -1181,6 +1195,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             tool_calls=tool_calls,
             optional_params=optional_params,
             kwargs=kwargs,
+            logging_obj=logging_obj,
         )
         return AgenticLoopPlan(
             run_agentic_loop=True,
@@ -1195,6 +1210,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         tool_calls: list[dict],
         optional_params: dict,
         kwargs: dict,
+        logging_obj: "LiteLLMLoggingObj | None",
     ) -> AgenticLoopRequestPatch:
         """Execute litellm.asearch() and build a Responses API rerun patch."""
         search_tasks: Final = [
@@ -1254,9 +1270,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             }
         }
 
-        full_model_name = model
-        if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str):
-            full_model_name = f"{kwargs['custom_llm_provider']}/{model}"
+        full_model_name: Final = _caller_model(model, logging_obj, kwargs)
 
         verbose_logger.debug(
             "WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d",
@@ -1814,6 +1828,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             tool_calls=tool_calls,
             optional_params=dict[str, object](optional_params),
             kwargs=dict[str, object](kwargs),
+            logging_obj=logging_obj,
             response_format=response_format,
         )
         if request_patch.messages is None:
@@ -1836,7 +1851,8 @@ class WebSearchInterceptionLogger(CustomLogger):
         messages: list[dict],
         tool_calls: list[dict],
         optional_params: dict,
-        kwargs: dict,
+        kwargs: dict[str, object],
+        logging_obj: "LiteLLMLoggingObj | None",
         response_format: str = "openai",
     ) -> AgenticLoopRequestPatch:
         """Execute litellm.search() and build chat-completion rerun patch."""
@@ -1922,11 +1938,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params
         }
 
-        full_model_name = model
-        if "custom_llm_provider" in kwargs:
-            custom_llm_provider: Final = kwargs["custom_llm_provider"]
-            if not model.startswith(custom_llm_provider) and "/" not in model:
-                full_model_name = f"{custom_llm_provider}/{model}"
+        full_model_name: Final = _caller_model(model, logging_obj, kwargs)
 
         verbose_logger.debug(
             "WebSearchInterception: Built chat completion request patch model=%s messages=%d",

litellm/interactions/litellm_responses_transformation/handler.py

@@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler:
         return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
             responses_response=responses_api_response,
             model=model,
+            custom_llm_provider=custom_llm_provider,
         )
 
     async def async_interactions_api_handler(
@@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler:
         return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
             responses_response=responses_api_response,
             model=model,
+            custom_llm_provider=responses_request.get("custom_llm_provider"),
         )

litellm/interactions/litellm_responses_transformation/transformation.py

@@ -12,6 +12,7 @@ from typing import Final, cast
 
 from pydantic import BaseModel
 
+from litellm.litellm_core_utils.core_helpers import billed_usage_cost
 from litellm.types.interactions import (
     InteractionInput,
     InteractionsAPIOptionalRequestParams,
@@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig:
     def transform_responses_response_to_interactions_response(
         responses_response: ResponsesAPIResponse,
         model: str | None = None,
+        custom_llm_provider: str | None = None,
     ) -> InteractionsAPIResponse:
         """
         Transform a Responses API response to an Interactions API response.
@@ -266,9 +268,11 @@ class LiteLLMResponsesInteractionsConfig:
         # (total_input_tokens, total_output_tokens)
         usage: Final[object] = getattr(responses_response, "usage", None)
         if usage:
+            reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider)
             interactions_response_dict["usage"] = {
                 "total_input_tokens": getattr(usage, "input_tokens", 0),
                 "total_output_tokens": getattr(usage, "output_tokens", 0),
+                **({} if reported_cost is None else {"cost": reported_cost}),
             }
 
         # Add updated (same as created for now)

litellm/litellm_core_utils/chat_completion_agentic_loop.py

@@ -15,6 +15,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
     DEFAULT_MAX_AGENTIC_LOOPS,
     validated_max_agentic_loops,
 )
+from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
 from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
 from litellm.types.integrations.custom_logger import (
@@ -182,9 +183,7 @@ async def _execute_chat_completion_agentic_plan(
     if patch.messages is None:
         raise ValueError("Agentic loop plan missing patched messages")
 
-    full_model_name = patch.model or model
-    if "/" not in full_model_name:
-        full_model_name = f"{custom_llm_provider}/{full_model_name}"
+    full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider)
 
     optional_params_for_followup: Final = {**optional_params, **patch.optional_params}
     if patch.tools is not None:

litellm/litellm_core_utils/core_helpers.py

@@ -35,6 +35,49 @@ def is_codex_user_agent(user_agent: str) -> bool:
     return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent))
 
 
+def qualify_provider_stripped_model(model: str, custom_llm_provider: str) -> str:
+    """Put the provider prefix back on a provider-stripped model."""
+    if not custom_llm_provider or model.startswith(f"{custom_llm_provider}/"):
+        return model
+    if _already_routes_to(model, custom_llm_provider):
+        return model
+    return f"{custom_llm_provider}/{model}"
+
+
+def _already_routes_to(model: str, custom_llm_provider: str) -> bool:
+    from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
+
+    return _leads_with_provider(model) and inferred_provider(model) == custom_llm_provider
+
+
+def _leads_with_provider(model: str) -> bool:
+    leading_segment, separator, _ = model.partition("/")
+    return bool(separator) and names_a_provider(leading_segment)
+
+
+def names_a_provider(candidate: str) -> bool:
+    """Whether a model name's leading segment is a provider rather than part of the model."""
+    import litellm
+    from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider
+    from litellm.llms.openai_like.json_loader import JSONProviderRegistry
+
+    provider_names: Final = {getattr(provider, "value", provider) for provider in litellm.provider_list}
+    return (
+        candidate in provider_names
+        or JSONProviderRegistry.exists(candidate)
+        or is_registered_custom_provider(candidate)
+    )
+
+
+def qualify_agentic_followup_model(patch_model: str | None, model: str, custom_llm_provider: str) -> str:
+    """Resolve the model an agentic follow-up re-dispatches, keeping a hook's own provider."""
+    if not patch_model or patch_model == model:
+        return qualify_provider_stripped_model(model, custom_llm_provider)
+    if _leads_with_provider(patch_model):
+        return patch_model
+    return qualify_provider_stripped_model(patch_model, custom_llm_provider)
+
+
 def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None:
     """
     Safely divide seconds by denominator, handling zero division.
@@ -985,6 +1028,16 @@ def chunk_usage(chunk: object) -> object:
     return getattr(chunk, "usage", None)
 
 
+USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
+
+
+def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
+    if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
+        return None
+    cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
+    return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None
+
+
 _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"})
 
 

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
 )
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
 from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
+from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
 from litellm.types.utils import LlmProviders
 
 from ..exceptions import (
@@ -2762,6 +2763,8 @@ def exception_type(
                     exception_provider=exception_provider,
                     extra_information=extra_information,
                 )
+            if custom_llm_provider == "bigmodel":
+                raise_bigmodel_business_error(model, f"{exception_provider} - {error_str}", original_exception)  # pyright: ignore[reportUnknownArgumentType]  # exception_type takes untyped parameters
             if custom_llm_provider == "openrouter":
                 _map_openrouter_exception(
                     model=model,

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -794,6 +794,13 @@ def _get_openai_compatible_provider_info(
             api_base,
             dynamic_api_key,
         ) = litellm.XAIChatConfig().get_openai_compatible_provider_info(api_base, api_key)
+    elif custom_llm_provider == "bigmodel":
+        from litellm.llms.bigmodel.common_utils import get_bigmodel_api_key, get_bigmodel_url
+
+        (
+            api_base,  # rebind-ok: existing provider-resolution pipeline resolves the API base in place
+            dynamic_api_key,  # rebind-ok: existing provider-resolution pipeline resolves the API key in place
+        ) = get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
     elif custom_llm_provider == "zai":
         (
             api_base,

litellm/litellm_core_utils/internal_call_metadata.py

@@ -120,7 +120,7 @@ def is_unbilled_non_inference_call(
     origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) if metadata is not None else None
     if origin == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN:
         return False
-    if not is_background_response(response):
+    if not is_background_response(_finished_response(response)):
         return True
     return origin == BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN or _is_managed_background_call(metadata)
 

litellm/litellm_core_utils/litellm_logging.py

@@ -23,7 +23,7 @@ from httpx import Response, Timeout
 from pydantic import BaseModel, JsonValue, TypeAdapter
 
 import litellm
-from litellm._internal_context import post_response_phase
+from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase
 from litellm._logging import (
     is_debugging_on,
     redact_string,
@@ -164,9 +164,10 @@ from litellm.types.llms.openai import (
 from litellm.types.mcp import MCPPostCallResponseObject
 from litellm.types.prompts.init_prompts import PromptSpec
 from litellm.types.rerank import RerankResponse
-from litellm.types.router import RouterRateLimitError
+from litellm.types.router import RouterRateLimitError, SilentExperimentFlag
 from litellm.types.utils import (
     DEPLOYMENT_SCOPED_PRICING_FIELDS,
+    AgenticLoopParams,
     CachingDetails,
     CallTypes,
     CostBreakdown,
@@ -316,7 +317,7 @@ else:
 _in_memory_loggers: Final[list[CustomLogger]] = []
 
 _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(
-    ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY)
+    ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment")
 )
 _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = (
     frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS
@@ -810,6 +811,7 @@ class Logging(LiteLLMLoggingBaseClass):
             litellm_params = scrub_sensitive_keys_in_metadata(litellm_params)
 
         self.litellm_params = litellm_params
+        self.is_silent_experiment: Final = in_mirrored_call()
 
         # Initialize cost breakdown field
         self.cost_breakdown: CostBreakdown | None = None
@@ -1160,6 +1162,9 @@ class Logging(LiteLLMLoggingBaseClass):
         if "custom_llm_provider" in self.model_call_details:
             self.custom_llm_provider = self.model_call_details["custom_llm_provider"]
 
+    def record_agentic_loop_params(self, params: AgenticLoopParams) -> None:
+        self.model_call_details["agentic_loop_params"] = params
+
     def record_applied_timeout(self, timeout: float | Timeout) -> None:
         self.applied_timeout = timeout
         self.litellm_params["timeout"] = timeout
@@ -1939,10 +1944,10 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def set_cost_breakdown(
         self,
-        input_cost: float,
-        output_cost: float,
+        input_cost: float | None,
+        output_cost: float | None,
         total_cost: float,
-        cost_for_built_in_tools_cost_usd_dollar: float,
+        cost_for_built_in_tools_cost_usd_dollar: float | None,
         additional_costs: dict | None = None,
         original_cost: float | None = None,
         discount_percent: float | None = None,
@@ -1981,14 +1986,17 @@ class Logging(LiteLLMLoggingBaseClass):
 
         self.billed_token_rates = billed_token_rates
         self.cost_breakdown = CostBreakdown(
-            input_cost=input_cost,
-            output_cost=output_cost,
             total_cost=total_cost,
-            tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar,
             service_tier=service_tier,
             data_residency=data_residency,
             vertex_location=vertex_location,
         )
+        if input_cost is not None:
+            self.cost_breakdown["input_cost"] = input_cost
+        if output_cost is not None:
+            self.cost_breakdown["output_cost"] = output_cost
+        if cost_for_built_in_tools_cost_usd_dollar is not None:
+            self.cost_breakdown["tool_usage_cost"] = cost_for_built_in_tools_cost_usd_dollar
         if cache_read_cost is not None and cache_read_cost > 0:
             self.cost_breakdown["cache_read_cost"] = cache_read_cost
         if cache_creation_cost is not None and cache_creation_cost > 0:
@@ -3260,7 +3268,7 @@ class Logging(LiteLLMLoggingBaseClass):
         """Restores trace_id/session_id contextvars once this attempt's own success
         logging (including any nested calls its callbacks trigger) is fully done."""
         try:
-            with post_response_phase():
+            with post_response_phase(), outside_mirrored_call():
                 return self._success_handler_body(
                     result=result,
                     start_time=start_time,
@@ -3760,7 +3768,7 @@ class Logging(LiteLLMLoggingBaseClass):
         """Restores trace_id/session_id contextvars once this attempt's own success
         logging (including any nested calls its callbacks trigger) is fully done."""
         try:
-            with post_response_phase():
+            with post_response_phase(), outside_mirrored_call():
                 return await self._async_success_handler_body(
                     result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs
                 )
@@ -4277,13 +4285,14 @@ class Logging(LiteLLMLoggingBaseClass):
         ``for_async_request`` marks the sync run of a failure whose async run also logs it, so a
         CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook."""
         try:
-            return self._failure_handler_body(
-                exception=exception,
-                traceback_exception=traceback_exception,
-                start_time=start_time,
-                end_time=end_time,
-                for_async_request=for_async_request,
-            )
+            with outside_mirrored_call():
+                return self._failure_handler_body(
+                    exception=exception,
+                    traceback_exception=traceback_exception,
+                    start_time=start_time,
+                    end_time=end_time,
+                    for_async_request=for_async_request,
+                )
         finally:
             self._restore_correlation_context()
 
@@ -4476,12 +4485,13 @@ class Logging(LiteLLMLoggingBaseClass):
         """Restores trace_id/session_id contextvars once this attempt's own failure
         logging (including any nested calls its callbacks trigger) is fully done."""
         try:
-            return await self._async_failure_handler_body(
-                exception=exception,
-                traceback_exception=traceback_exception,
-                start_time=start_time,
-                end_time=end_time,
-            )
+            with outside_mirrored_call():
+                return await self._async_failure_handler_body(
+                    exception=exception,
+                    traceback_exception=traceback_exception,
+                    start_time=start_time,
+                    end_time=end_time,
+                )
         finally:
             self._restore_correlation_context()
 
@@ -6570,6 +6580,7 @@ class StandardLoggingPayloadSetup:
         start_time: dt_object | None = None,
         response_id: str | None = None,
         custom_llm_provider: str | None = None,
+        is_silent_experiment: bool = False,
     ) -> StandardLoggingMetadata:
         """
         Clean and filter the metadata dictionary to include only the specified keys in StandardLoggingMetadata.
@@ -6643,6 +6654,7 @@ class StandardLoggingPayloadSetup:
                 proxy_stamped_used_client_oauth_token(metadata, litellm_params),
                 custom_llm_provider,
             ),
+            **(SilentExperimentFlag(is_silent_experiment=True) if is_silent_experiment else SilentExperimentFlag()),
         )
         if isinstance(metadata, dict):
             for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS:
@@ -7443,6 +7455,7 @@ def get_standard_logging_object_payload(
             start_time=start_time,
             response_id=id,
             custom_llm_provider=selected_provider if isinstance(selected_provider, str) else None,
+            is_silent_experiment=logging_obj.is_silent_experiment is True,
         )
         _request_body: Final = proxy_server_request.get("body", {})
         end_user_id: Final = clean_metadata["user_api_key_end_user_id"] or _request_body.get(
@@ -7735,9 +7748,8 @@ def get_standard_logging_metadata(
     )
     if isinstance(metadata, dict):
         # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields
-        for key in StandardLoggingMetadata.__annotations__:
-            if key in metadata:
-                clean_metadata[key] = metadata[key]
+        for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - {"is_silent_experiment"}:
+            clean_metadata[key] = metadata[key]
 
         if metadata.get("user_api_key") is not None:
             if is_valid_sha256_hash(str(metadata.get("user_api_key"))):

litellm/litellm_core_utils/llm_request_utils.py

@@ -1,8 +1,9 @@
-from collections.abc import Mapping
+from collections.abc import Iterable, Iterator, Mapping
 from typing import Final, cast
 
 import litellm
 from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
+from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
 
 
 def _form_field_value(value: object) -> str:
@@ -153,24 +154,33 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
     if custom_llm_provider not in litellm.models_by_provider:
         return []
 
-    known_models: Final = litellm.models_by_provider.get(custom_llm_provider, [])
-    model_costs: Final = []
+    known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
+    ranked_rows: Final = (
+        (route_rank, cost, model)
+        for model, cost in _chat_row_costs(known_models, custom_llm_provider)
+        if (route_rank := _route_rank(model, custom_llm_provider)) is not None
+    )
+    return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
 
-    for model in known_models:
+
+def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
+    for model in models:
         try:
             model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
         except Exception:
             continue
         if model_info.get("mode") != "chat":
             continue
-        _cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
-        model_costs.append((model, _cost))
+        yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
 
-    # Sort by cost (ascending)
-    model_costs.sort(key=lambda x: x[1])
 
-    # Return the top n cheapest models
-    return [model for model, _ in model_costs[:n]]
+def _route_rank(model: str, custom_llm_provider: str) -> int | None:
+    routed_provider: Final = inferred_provider(model)
+    if routed_provider is None:
+        return 1
+    if routed_provider == inferred_provider(f"{custom_llm_provider}/{model}"):
+        return 0
+    return None
 
 
 def get_proxy_server_request_headers(litellm_params: dict | None) -> dict:

litellm/litellm_core_utils/llm_response_utils/response_metadata.py

@@ -27,6 +27,15 @@ def _timing_window_start(
     return start_time, False
 
 
+def response_ms_since_logging_start(logging_obj: LiteLLMLoggingObject) -> float | None:
+    reported_ms: Final = logging_obj.response_timing_metrics.get("_response_ms")
+    if reported_ms is None:
+        return None
+    logging_start: Final = logging_obj.start_time
+    window_start: Final = _timing_window_start(logging_start, logging_obj)[0]
+    return reported_ms - max(0.0, logging_start.timestamp() - window_start.timestamp()) * 1000
+
+
 def _union_duration_ms(windows: object, lower: float, upper: float) -> float | None:
     if not isinstance(windows, (list, tuple)):
         return None

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -17,7 +17,7 @@ from pathlib import Path
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
 
-from pydantic import TypeAdapter
+from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm import verbose_logger
@@ -2073,6 +2073,19 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
     return cast(_MarkedT, marked)  # cast-ok: same block shape as the input plus the marker key
 
 
+_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
+
+
+def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
+    if marker is None or not drop_params:
+        return marker
+    try:
+        return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
+    except ValidationError:
+        verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
+        return None
+
+
 def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
     """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block."""
     if block.get("cache_control") is None:

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -12,12 +12,18 @@ from typing_extensions import ReadOnly, Required
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.litellm_core_utils.core_helpers import chunk_usage, finite_cost, provider_reported_usage_cost
+from litellm.litellm_core_utils.core_helpers import (
+    USAGE_COST_BILLED_PROVIDERS,
+    chunk_usage,
+    finite_cost,
+    provider_reported_usage_cost,
+)
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_content_from_model_response,  # pyright: ignore[reportUnknownVariableType]  # response param accepts an untyped dict
 )
+from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionAssistantContentValue,
@@ -51,7 +57,7 @@ from litellm.types.utils import (
     UnreportedUsage,
     Usage,
 )
-from litellm.utils import print_verbose, token_counter
+from litellm.utils import get_base_model_from_metadata, print_verbose, token_counter
 
 if TYPE_CHECKING:
     from openai.types.completion_usage import CompletionUsage
@@ -1353,7 +1359,7 @@ def stream_chunk_builder_text_completion(
     return TextCompletionResponse(**response)
 
 
-_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value})
+_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
 
 
 def _logged_provider(logging_obj: Optional["Logging"]) -> object:
@@ -1554,6 +1560,9 @@ def stream_chunk_builder(
             return stream_chunk_builder_text_completion(chunks=chunks, messages=messages)
 
         model: Final = chunks[0]["model"]
+        usage_model: Final = usage_tokenizer_model(
+            model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
+        )
         # Initialize the response dictionary
         response: Final = processor.build_base_response(chunks)
 
@@ -1577,7 +1586,7 @@ def stream_chunk_builder(
             completion_output = get_content_from_model_response(response)
             usage = processor.calculate_usage(
                 chunks=chunks,
-                model=model,
+                model=usage_model,
                 completion_output=completion_output,
                 messages=messages,
                 reasoning_tokens=0,
@@ -1768,7 +1777,7 @@ def stream_chunk_builder(
 
         usage = processor.calculate_usage(
             chunks=chunks,
-            model=model,
+            model=usage_model,
             completion_output=completion_output,
             messages=messages,
             reasoning_tokens=reasoning_tokens,

litellm/litellm_core_utils/streaming_handler.py

@@ -1938,6 +1938,16 @@ class CustomStreamWrapper:
         model_response: Final = self.model_response_creator(
             hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
         )
+        served_tier: Final = next(
+            (
+                tier
+                for chunk in reversed(self.chunks)
+                if isinstance(tier := getattr(chunk, "service_tier", None), str) and tier
+            ),
+            None,
+        )
+        if served_tier is not None:
+            setattr(model_response, "service_tier", served_tier)  # noqa: B010  # pydantic extra, not a declared field
         _finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason
         if _finish_reason is not None:
             model_response.choices[0].finish_reason = _finish_reason

litellm/litellm_core_utils/token_counter.py

@@ -15,6 +15,7 @@ import anyio
 import anyio.lowlevel
 import httpx
 import tiktoken
+from tiktoken.model import MODEL_TO_ENCODING
 from typing_extensions import ParamSpec, TypeVar
 
 import litellm
@@ -49,6 +50,7 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace
 from litellm.litellm_core_utils.url_utils import safe_get
 from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value
+from litellm.router_utils.deployment_pricing import deployment_pricing_models
 from litellm.rust_bridge.tokenizer import get_encoding
 from litellm.types.llms.anthropic import (
     AnthropicContentParamSource,
@@ -609,7 +611,7 @@ T_ParamSpec: Final = ParamSpec("T_ParamSpec")
 T_Retval = TypeVar("T_Retval")
 _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter")
 _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset(
-    ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming")
+    ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming", "custom")
 )
 
 
@@ -1005,17 +1007,144 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
 
 
 def openai_tokenizer_encoding_name(model: str) -> str:
-    """The tiktoken encoding name for `model`, without loading the encoding."""
-    from litellm.utils import print_verbose
+    """The tiktoken encoding name for `model`, without loading the encoding.
 
-    model_to_use: Final = _fix_model_name(model)
+    A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does
+    know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every
+    OpenAI model since gpt-4o uses. Provider and region prefixes (`azure/eu/`, `azure_ai/`) are dropped, a name
+    listed only under Azure (`gpt-41`, `openai/o1-mini`) counts as listed through its `azure/` cost-map key, and an
+    unlisted name tiktoken counts with o200k_base (`openai/gpt-5.9`) or named like a gpt-5 or o-series release
+    (`openai/gpt-6.2`) gets it."""
+    model_to_use: Final = _openai_model_name(model) or _fix_model_name(model)
     if "gpt-4o" in model_to_use:
         return "o200k_base"
     try:
         return tiktoken.encoding_name_for_model(model_to_use)
     except KeyError:
-        print_verbose("Warning: model not found. Using cl100k_base encoding.")
-        return "cl100k_base"
+        return _suffixed_variant_encoding_name(model_to_use) or "o200k_base"
+
+
+def unresolved_tokenizer_model(model: str) -> str:
+    """`model`, or `gpt-3.5-turbo` when resolving the OpenAI model `model` names moves it off cl100k_base, the encoding
+    tiktoken gives the name as written, so a count with the result never drops below that one."""
+    as_written: Final = _fix_model_name(model)
+    as_written_encoding: Final = (
+        "o200k_base" if "gpt-4o" in as_written else (_tiktoken_encoding_name(as_written) or "cl100k_base")
+    )
+    moved_off_cl100k: Final = (
+        as_written_encoding == "cl100k_base" and openai_tokenizer_encoding_name(model) != "cl100k_base"
+    )
+    return "gpt-3.5-turbo" if moved_off_cl100k else model
+
+
+def usage_tokenizer_model(model: str, base_model: str | None) -> str:
+    """The model to estimate missing usage with: a deployment's `base_model` when it names an OpenAI model, since a
+    custom deployment name says nothing about its tokenizer, else `model`."""
+    return base_model if base_model is not None and is_openai_model(base_model) else model
+
+
+def is_openai_model(model: str) -> bool:
+    """Whether `model` names an OpenAI or Azure OpenAI model, listed in the cost map or counted with o200k_base by
+    tiktoken, with or without its prefixes."""
+    return _openai_model_name(model) is not None
+
+
+def deployment_openai_model(deployment: object) -> str | None:
+    """The OpenAI model a router deployment serves, read from its base model first, since a custom deployment name
+    (`azure/my-gpt-5`) says nothing about its tokenizer, or None when it serves no OpenAI model."""
+    return _as_openai_model(_served_model(deployment))
+
+
+def served_openai_models(deployments: Iterable[object]) -> tuple[str | None, ...]:
+    """The distinct OpenAI models `deployments` serve, with None standing for any deployment serving another model."""
+    served: Final = dict.fromkeys(_served_model(deployment) for deployment in deployments)
+    return tuple(dict.fromkeys(_as_openai_model(model) for model in served))
+
+
+def _served_model(deployment: object) -> str | None:
+    return next(iter(deployment_pricing_models(deployment)), None)
+
+
+def _as_openai_model(model: str | None) -> str | None:
+    return model if model is not None and is_openai_model(model) else None
+
+
+def single_encoding_openai_model(served: Sequence[str | None]) -> str | None:
+    """An OpenAI model to count a request with, when every `served_openai_models` entry is an OpenAI model on the same
+    encoding, else None."""
+    if not served or None in served:
+        return None
+    encodings: Final = frozenset(openai_tokenizer_encoding_name(model) for model in served if model is not None)
+    return served[0] if len(encodings) == 1 else None
+
+
+def _openai_model_name(model: str) -> str | None:
+    return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model)
+
+
+_VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)")
+
+
+def _versioned_openai_model_name(model: str) -> str | None:
+    name: Final = model.rsplit("/", 1)[-1]
+    return name if _VERSIONED_OPENAI_MODEL.match(name) else None
+
+
+def _tiktoken_o200k_model_name(model: str) -> str | None:
+    name: Final = model.rsplit("/", 1)[-1]
+    return name if _tiktoken_encoding_name(name) == "o200k_base" else None
+
+
+def _listed_openai_model_name(model: str) -> str | None:
+    segments: Final = model.split("/")
+    suffixes: Final = ("/".join(segments[start:]) for start in range(len(segments)))
+    listed: Final = any(
+        _is_openai_cost_map_model(suffix) or _is_openai_cost_map_model(f"azure/{suffix}") for suffix in suffixes
+    )
+    return segments[-1] if listed else None
+
+
+def _is_openai_cost_map_model(model: str) -> bool:
+    return (
+        model in litellm.azure_llms
+        or model in litellm.open_ai_chat_completion_models
+        or _is_azure_openai_text_model(model)
+    )
+
+
+_TEXT_MODEL_MODES: Final = frozenset({"chat", "completion", "responses", "realtime"})
+
+
+def _is_azure_openai_text_model(model: str) -> bool:
+    cost_info: Final[Mapping[str, object]] = litellm.model_cost.get(model) or {}
+    return (
+        cost_info.get("litellm_provider") == "azure"
+        and cost_info.get("mode") in _TEXT_MODEL_MODES
+        and _is_openai_model_name(model.rsplit("/", 1)[-1])
+    )
+
+
+def _is_openai_model_name(name: str) -> bool:
+    return (
+        name in litellm.open_ai_chat_completion_models
+        or _tiktoken_encoding_name(name) is not None
+        or any(
+            name.startswith(f"{listed}-") or listed.startswith(f"{name}-")
+            for listed in tuple(litellm.open_ai_chat_completion_models)
+        )
+    )
+
+
+def _tiktoken_encoding_name(model: str) -> str | None:
+    try:
+        return tiktoken.encoding_name_for_model(model)
+    except KeyError:
+        return None
+
+
+def _suffixed_variant_encoding_name(model: str) -> str | None:
+    known: Final = tuple(name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-"))
+    return MODEL_TO_ENCODING[max(known, key=len)] if known else None
 
 
 def uses_legacy_message_accounting(model: str) -> bool:

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -432,27 +432,30 @@ class AnthropicMessagesHandler(BaseTranslation):
         message, _ = serialize_http_exception_detail(exc.detail)
         return tuple(anthropic_sse_error_frames(message))
 
-    def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]:
+    def build_block_response(self, exc: "ModifyResponseException") -> "AnthropicMessagesResponse":
         import uuid
 
-        from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
-            FakeAnthropicMessagesStreamIterator,
-        )
         from litellm.llms.base_llm.guardrail_translation.utils import (
             blocked_response_usage,
         )
         from litellm.types.utils import AnthropicMessagesResponse
 
-        block_response: Final = AnthropicMessagesResponse(
+        return AnthropicMessagesResponse(
             id=f"msg_{uuid.uuid4()}",
             type="message",
             role="assistant",
             content=[{"type": "text", "text": exc.message}],
             model=exc.model,
             stop_reason="end_turn",
-            usage=blocked_response_usage(getattr(exc, "original_response", None)),
+            usage=blocked_response_usage(exc.original_response),
         )
-        return list(FakeAnthropicMessagesStreamIterator(response=block_response))
+
+    def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]:
+        from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
+            FakeAnthropicMessagesStreamIterator,
+        )
+
+        return list(FakeAnthropicMessagesStreamIterator(response=self.build_block_response(exc)))
 
     def _block_continuation_chunks(
         self, exc: "ModifyResponseException", responses_so_far: Sequence[object]

litellm/llms/anthropic/chat/handler.py

@@ -17,6 +17,7 @@ import litellm
 import litellm.litellm_core_utils
 import litellm.types
 import litellm.types.utils
+from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body
 from litellm.anthropic_beta_headers_manager import (
     update_request_with_filtered_beta,
 )
@@ -506,11 +507,20 @@ class AnthropicChatCompletion(BaseLLM):
             The pair stays mutable because the streaming path rewrites it in
             place (`data["stream"] = True`) before sending.
             """
-            request_headers, data = update_request_with_filtered_beta(
+            request_headers, filtered_data = update_request_with_filtered_beta(
                 headers=headers,
                 request_data=request_data,
                 provider=custom_llm_provider,
             )
+            data: Final = dict(
+                strip_beta_gated_request_body(
+                    filtered_data,
+                    request_headers,
+                    drop_params=litellm_params.get("drop_params"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # litellm_params is the handler's bare dict
+                    model=model,
+                    provider=custom_llm_provider,
+                )
+            )
             logging_obj.pre_call(
                 input=messages,
                 api_key=api_key,

litellm/llms/anthropic/chat/transformation.py

@@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm.constants import (
+    ANTHROPIC_BILLING_METADATA_PREFIX,
     ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
     ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES,
     DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS,
@@ -2066,7 +2067,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                     if not system_message_block["content"]:
                         continue
                     if self.should_strip_billing_metadata() and system_message_block["content"].startswith(
-                        "x-anthropic-billing-header:"
+                        ANTHROPIC_BILLING_METADATA_PREFIX
                     ):
                         continue
                     anthropic_system_message_content = AnthropicSystemMessageContent(
@@ -2086,7 +2087,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                             self.should_strip_billing_metadata()
                             and _content.get("type") == "text"
                             and text_value
-                            and text_value.startswith("x-anthropic-billing-header:")
+                            and text_value.startswith(ANTHROPIC_BILLING_METADATA_PREFIX)
                         ):
                             continue
                         anthropic_system_message_content = AnthropicSystemMessageContent(

litellm/llms/anthropic/common_utils.py

@@ -51,6 +51,7 @@ from litellm.types.llms.anthropic import (
     ANTHROPIC_OAUTH_BETA_HEADER,
     ANTHROPIC_OAUTH_TOKEN_PREFIX,
     ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX,
+    ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,
     ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
     ANTHROPIC_TOOL_SEARCH_TOOL_TYPES,
     AllAnthropicToolsValues,
@@ -121,6 +122,54 @@ _DECODED_JSON: Final = TypeAdapter(object)
 _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
 
 
+def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
+    if custom_llm_provider == "anthropic":
+        return True
+    model_lower: Final = model.lower()
+    if custom_llm_provider == "bedrock":
+        return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower)
+    if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai":
+        return "claude" in model_lower
+    return False
+
+
+def is_claude_model_target(custom_llm_provider: str | None, model: str) -> bool:
+    return "claude" in model.lower() and is_claude_tool_target(custom_llm_provider, model)
+
+
+def _hoistable_defer_loading(tool_dict: Mapping[object, object], custom: object) -> bool | None:
+    if not isinstance(custom, dict) or "defer_loading" in tool_dict:
+        return None
+    defer_loading: Final = OBJECT_MAPPING_ADAPTER.validate_python(custom).get("defer_loading")
+    return defer_loading if isinstance(defer_loading, bool) else None
+
+
+def _tool_without_custom_field(tool: object, hoist_defer_loading: bool, drop_custom: bool) -> object:
+    if not isinstance(tool, dict):
+        return tool
+
+    if "custom" not in tool:
+        return tool
+
+    tool_dict: Final = OBJECT_MAPPING_ADAPTER.validate_python(tool)
+
+    custom: Final = tool_dict["custom"]
+    without_custom: Final = {key: value for key, value in tool_dict.items() if key != "custom"}
+    defer_loading: Final = _hoistable_defer_loading(tool_dict, custom) if hoist_defer_loading else None
+    if defer_loading is None:
+        return without_custom if drop_custom else tool_dict
+
+    unhoisted: Final = {
+        key: value for key, value in OBJECT_MAPPING_ADAPTER.validate_python(custom).items() if key != "defer_loading"
+    }
+    kept_custom: Final = {"custom": unhoisted} if unhoisted and not drop_custom else {}
+    return {**without_custom, "defer_loading": defer_loading, **kept_custom}
+
+
+def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: bool, drop_custom: bool) -> list[object]:
+    return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
+
+
 def is_anthropic_messages_url(url: str) -> bool:
     """Check whether a URL addresses Anthropic's Messages API."""
     parsed_url: Final = urlparse(url)
@@ -552,6 +601,20 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             return False
         return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates"
 
+    def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]:
+        return (
+            *(
+                (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,)
+                if self.is_thinking_display_updates_used(thinking)
+                else ()
+            ),
+            *(
+                (ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,)
+                if isinstance(thinking, dict) and "block_binding" in thinking
+                else ()
+            ),
+        )
+
     def is_mid_conversation_tool_change_used(self, messages: Sequence[object]) -> bool:
         for message in messages:
             if message_field(message, "role") != "system":
@@ -1132,7 +1195,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         *,
         custom_llm_provider: str,
         is_mid_conversation_output_config_used: bool = False,
-        is_thinking_display_updates_used: bool = False,
+        thinking: AnthropicThinkingParam | None = None,
         is_mid_conversation_tool_change_used: bool = False,
     ) -> list[str]:
         """
@@ -1169,13 +1232,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         if is_mid_conversation_output_config_used:
             betas.append(ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER)
 
-        thinking_display_betas: Final = (
-            (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else ()
-        )
         tool_change_betas: Final = (
             (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else ()
         )
-        return list(set(betas).union(thinking_display_betas, tool_change_betas))
+        return list(set(betas).union(self.thinking_betas(thinking), tool_change_betas))
 
     @staticmethod
     def _make_api_key_auth_header(
@@ -1237,7 +1297,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         api_base: str | None = None,
         use_bearer_for_custom_base: bool = False,
         is_mid_conversation_output_config_used: bool = False,
-        is_thinking_display_updates_used: bool = False,
+        thinking: AnthropicThinkingParam | None = None,
         is_mid_conversation_tool_change_used: bool = False,
         wif_minted: bool = False,
     ) -> dict:
@@ -1298,7 +1358,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             betas.update(user_anthropic_beta_headers)
 
         all_betas: Final = betas.union(
-            (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,) if is_thinking_display_updates_used else (),
+            self.thinking_betas(thinking),
             (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else (),
         )
 
@@ -1385,7 +1445,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             auth_token=auth_token,
             file_id_used=file_id_used,
             is_mid_conversation_output_config_used=is_mid_conversation_output_config_used,
-            is_thinking_display_updates_used=self.is_thinking_display_updates_used(optional_params.get("thinking")),
+            thinking=optional_params.get("thinking"),
             is_mid_conversation_tool_change_used=self.is_mid_conversation_tool_change_used(messages),
             web_search_tool_used=web_search_tool_used,
             is_vertex_request=optional_params.get("is_vertex_request", False),

litellm/llms/anthropic/count_tokens/handler.py

@@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
         tools: list[dict[str, JsonValue]] | None = None,
         system: JsonValue = None,
         optional_params: Mapping[str, JsonValue] | None = None,
+        drop_params: bool = False,
     ) -> dict[str, JsonValue]:
         """
         Handle a CountTokens request using httpx.
@@ -69,6 +70,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
                 tools=tools,
                 system=system,
                 optional_params=optional_params,
+                drop_params=drop_params,
             )
 
             verbose_logger.debug("Transformed request: %s", request_body)

litellm/llms/anthropic/count_tokens/token_counter.py

@@ -11,6 +11,7 @@ from pydantic import JsonValue
 import litellm
 from litellm._logging import verbose_logger
 from litellm.exceptions import AuthenticationError
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
 from litellm.llms.anthropic.count_tokens.transformation import (
     AnthropicCountFields,
@@ -131,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
                 tools=tools,
                 system=system,
                 optional_params=count_options,
+                drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
             )
 
             if result is not None:

litellm/llms/anthropic/count_tokens/transformation.py

@@ -26,7 +26,11 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     split_leading_system_run,
 )
 from litellm.litellm_core_utils.url_utils import join_api_base
-from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
+from litellm.llms.anthropic.common_utils import (
+    is_claude_model_target,
+    merge_anthropic_beta_headers,
+    tools_without_custom_field,
+)
 from litellm.llms.anthropic.wif import resolve_anthropic_base
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
 from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@@ -372,6 +376,7 @@ class AnthropicCountTokensConfig:
         tools: list[dict[str, JsonValue]] | None = None,
         system: JsonValue = None,
         optional_params: Mapping[str, JsonValue] | None = None,
+        drop_params: bool = False,
     ) -> dict[str, JsonValue]:  # mutable-ok: provider transport requires JSON dictionaries
         """
         Transform request to Anthropic CountTokens format.
@@ -391,7 +396,19 @@ class AnthropicCountTokensConfig:
                     **MappingProxyType(
                         {
                             key: value
-                            for key, value in (("system", counted_system), ("tools", tools))
+                            for key, value in (
+                                ("system", counted_system),
+                                (
+                                    "tools",
+                                    None
+                                    if tools is None
+                                    else tools_without_custom_field(
+                                        tools,
+                                        hoist_defer_loading=is_claude_model_target("anthropic", model),
+                                        drop_custom=drop_params,
+                                    ),
+                                ),
+                            )
                             if value is not None
                         }
                     ),

litellm/llms/anthropic/messages/adapters/transformation.py

@@ -203,7 +203,7 @@ from litellm.types.llms.openai import (
     ChatCompletionUserMessage,
     ToolMessageContentPart,
 )
-from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
+from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage
 from litellm.utils import supports_mid_conversation_system
 
 from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper
@@ -362,6 +362,9 @@ class AnthropicAdapter:
 _BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object])
 
 
+_CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.value})
+
+
 class LiteLLMAnthropicMessagesAdapter:
     def __init__(self):
         pass
@@ -817,6 +820,7 @@ class LiteLLMAnthropicMessagesAdapter:
             "name",
             "input_schema",
             "description",
+            "custom",
             "cache_control",
             "strict",
             "type",
@@ -1127,7 +1131,8 @@ class LiteLLMAnthropicMessagesAdapter:
         """Translate Anthropic thinking to either thinking or reasoning_effort.
 
         A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one
-        speaks that param. Carrying its adaptive effort tier alongside takes two different params,
+        speaks that param, except Kyma, which serves Claude over the OpenAI dialect and takes only
+        ``reasoning_effort``, so its Claude models get the non-Claude mapping. Carrying its adaptive effort tier alongside takes two different params,
         because the two are not interchangeable at the provider mapping below.
 
         Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking``
@@ -1158,7 +1163,9 @@ class LiteLLMAnthropicMessagesAdapter:
         is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model(
             model
         )
-        is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)
+        is_claude_target: Final = (
+            self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)
+        ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS
         output_config: Final = anthropic_message_request.get("output_config")
 
         if is_claude_target:

litellm/llms/anthropic/messages/native/transformation.py

@@ -2,10 +2,22 @@ from collections.abc import AsyncIterator, Mapping, Sequence
 from typing import ClassVar, Final
 
 import httpx
+from pydantic import TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config
+from litellm.anthropic_beta_gated_fields import (
+    log_beta_gated_strips,
+    strip_beta_gated_native_request,
+    strip_beta_gated_system_messages,
+    unstripped_native_request,
+)
+from litellm.anthropic_beta_headers_manager import (
+    get_unsupported_headers,
+    has_beta_headers_config,
+    is_beta_header_supported_by_bundled_config,
+)
+from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
 from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -19,7 +31,6 @@ from litellm.types.llms.anthropic import (
     ANTHROPIC_ADVISOR_TOOL_TYPE,
     ANTHROPIC_BETA_HEADER_VALUES,
     ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
-    ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
     AnthropicMessagesRequest,
 )
 from litellm.types.llms.anthropic_messages.anthropic_response import (
@@ -29,13 +40,16 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
 from litellm.types.router import GenericLiteLLMParams
 
 from ...common_utils import (
+    OBJECT_LIST_ADAPTER,
     AnthropicError,
     AnthropicModelInfo,
+    is_claude_model_target,
     merge_anthropic_beta_headers,
     optionally_handle_anthropic_oauth,
     requires_native_compaction_beta,
     strip_advisor_blocks_from_messages,
     strip_encrypted_reasoning_blocks_from_anthropic_messages,
+    tools_without_custom_field,
 )
 from .mid_conversation_system import (
     as_system_content_blocks,
@@ -47,6 +61,8 @@ DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01"
 
 _CALLER_CREDENTIAL_HEADERS: Final = frozenset({"x-api-key", "authorization"})
 
+_MESSAGE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
+
 
 def _carries_caller_credential(headers: Mapping[str, str]) -> bool:
     """Whether the caller sent their own Anthropic credential, in which case this passthrough
@@ -81,6 +97,9 @@ def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessa
     return response
 
 
+_MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
+
+
 class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     _workload_identity_eligible: ClassVar[bool] = True
 
@@ -181,6 +200,16 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         """
         return False
 
+    def hoists_wrapped_defer_loading(self, model: str) -> bool:
+        return is_claude_model_target(self._resolved_provider, model)
+
+    def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]:
+        return tools_without_custom_field(
+            tools,
+            hoist_defer_loading=self.hoists_wrapped_defer_loading(model),
+            drop_custom=drops_unsupported_params(drop_params),
+        )
+
     @staticmethod
     def _filter_billing_headers_from_system(system_param):
         """
@@ -194,7 +223,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         """
         if isinstance(system_param, str):
             # If it's a string and starts with billing header, filter it out
-            if system_param.startswith("x-anthropic-billing-header:"):
+            if system_param.startswith(ANTHROPIC_BILLING_METADATA_PREFIX):
                 return None
             return system_param
         elif isinstance(system_param, list):
@@ -205,7 +234,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
                     text = content_block.get("text", "")
                     content_type = content_block.get("type", "")
                     # Skip text blocks that start with billing header
-                    if content_type == "text" and text.startswith("x-anthropic-billing-header:"):
+                    if content_type == "text" and text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX):
                         continue
                     filtered_list.append(content_block)
                 else:
@@ -215,7 +244,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         else:
             return system_param
 
-    def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None:
+    def _strip_beta_gated_system_turns(
+        self, messages: Sequence[Mapping[str, object]], model: str, drop_params: object
+    ) -> Sequence[Mapping[str, object]]:
+        if not has_beta_headers_config(self._resolved_provider):
+            return messages
+        gated: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=drop_params)
+        log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider)
+        return _MESSAGE_MAPPINGS.validate_python(gated.messages) if gated.stripped else messages
+
+    def _normalize_system_role_messages(
+        self, anthropic_messages_request: dict, model: str, drop_params: object
+    ) -> None:
         """Normalize ``role: "system"`` entries in ``messages`` per the Anthropic
         ``/v1/messages`` contract, which the first-party API, Bedrock Invoke,
         Vertex, and Azure Foundry all enforce identically.
@@ -263,7 +303,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
                 custom_llm_provider=self.custom_llm_provider,
                 key="supports_mid_conversation_system",
             )
-            else list(convert_mid_conversation_system_turns(messages[leading_count:]))
+            else list(
+                convert_mid_conversation_system_turns(
+                    self._strip_beta_gated_system_turns(
+                        messages[leading_count:],  # pyright: ignore[reportUnknownArgumentType]  # messages comes from the bare request dict
+                        model=model,
+                        drop_params=drop_params,
+                    )
+                )
+            )
         )
         if hoisted or remaining != messages:
             anthropic_messages_request["messages"] = remaining
@@ -322,7 +370,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
                     model=model,
                 ),
             )
-        return self._finalize_messages_headers(headers, optional_params, messages), api_base
+        return self.finalize_messages_headers(headers, optional_params, messages), api_base
 
     async def avalidate_anthropic_messages_environment(
         self,
@@ -362,7 +410,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
                     model=model,
                 ),
             )
-        return self._finalize_messages_headers(oauth_headers, optional_params, messages), api_base
+        return self.finalize_messages_headers(oauth_headers, optional_params, messages), api_base
 
     def _require_auth_header(self, auth_header: Mapping[str, str] | None, model: str) -> Mapping[str, str]:
         if auth_header is None:
@@ -396,7 +444,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
 
         return config_allows_workload_identity(self)
 
-    def _finalize_messages_headers(
+    def finalize_messages_headers(
         self,
         headers: dict,  # mutable-ok: out-param
         optional_params: dict,  # mutable-ok: out-param
@@ -605,6 +653,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         if thinking_enabled or effort_enabled:
             optional_params.pop("temperature", None)
 
+    def strips_beta_gated_fields_from_final_headers(self) -> bool:
+        """False on legs whose final ``anthropic_beta`` list is only known after this transform (Bedrock Invoke),
+        which strip beta-gated fields themselves once that list exists, and on providers with no row in
+        ``anthropic_beta_headers_config.json``, whose filter drops every beta and so cannot say which fields fail."""
+        return has_beta_headers_config(self._resolved_provider)
+
     def transform_anthropic_messages_request(
         self,
         model: str,
@@ -700,20 +754,47 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
 
         # Auto-strip advisor blocks from history if advisor tool is absent.
         # Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
-        _tools: Final = anthropic_messages_optional_request_params.get("tools") or []
+        tools: Final = anthropic_messages_optional_request_params.get("tools")
+        _tools: Final = tools or []
         _has_advisor: Final = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _tools)
         if not _has_advisor:
             messages = strip_advisor_blocks_from_messages(messages)
 
+        gated: Final = (
+            strip_beta_gated_native_request(
+                headers,  # pyright: ignore[reportUnknownArgumentType]  # the base signature types this as a bare dict
+                anthropic_messages_optional_request_params,  # pyright: ignore[reportUnknownArgumentType]  # the base signature types this as a bare dict
+                messages,  # pyright: ignore[reportUnknownArgumentType]  # the base signature types this as a bare list
+                drop_params=getattr(litellm_params, "drop_params", None),
+            )
+            if self.strips_beta_gated_fields_from_final_headers()
+            else unstripped_native_request(anthropic_messages_optional_request_params, messages)  # pyright: ignore[reportUnknownArgumentType]  # the base signature types these as bare dicts
+        )
+        log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider)
         anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest(
             messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(
-                drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None))
+                drop_non_string_tool_result_texts(
+                    _MESSAGES_ADAPTER.validate_python(gated.messages) if gated.messages_stripped else messages,
+                    getattr(litellm_params, "drop_params", None),
+                )
             ),
             max_tokens=max_tokens,
             model=model,
             **anthropic_messages_optional_request_params,
         )
-        return dict(anthropic_messages_request)
+        request: Final = (
+            {**dict(anthropic_messages_request), "thinking": gated.thinking}
+            if gated.thinking_stripped
+            else dict(anthropic_messages_request)  # pyright: ignore[reportUnknownVariableType]  # built from the base signature's bare dict
+        )
+        if not isinstance(tools, list):
+            return request
+        return {
+            **request,
+            "tools": self.tools_for_upstream(
+                OBJECT_LIST_ADAPTER.validate_python(tools), model, litellm_params.drop_params
+            ),
+        }
 
     def transform_anthropic_messages_response(
         self,
@@ -844,17 +925,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         if AnthropicModelInfo().is_tool_search_used(tools):
             beta_values.add(get_tool_search_beta_header(custom_llm_provider))
 
-        thinking_display_betas: Final = (
-            (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,)
-            if AnthropicModelInfo().is_thinking_display_updates_used(optional_params.get("thinking"))
-            else ()
-        )
+        thinking_betas: Final = AnthropicModelInfo().thinking_betas(optional_params.get("thinking"))
         tool_change_betas: Final = (
             (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,)
             if AnthropicModelInfo().is_mid_conversation_tool_change_used(messages)
             else ()
         )
-        all_beta_values: Final = beta_values.union(thinking_display_betas, tool_change_betas)
+        all_beta_values: Final = beta_values.union(thinking_betas, tool_change_betas)
         dangerous_tool_use: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value
         forwarded_beta_values: Final = (  # pyright: ignore[reportUnknownVariableType]  # derived from the bare beta set
             all_beta_values - {dangerous_tool_use}

litellm/llms/anthropic/messages/responses_adapters/handler.py

@@ -15,7 +15,11 @@ from litellm.integrations.anthropic_cache_control_hook import (
     supports_openai_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
-from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    cache_control_as_prompt_cache_breakpoint,
+    prompt_cache_breakpoint_for_wire,
+    with_prompt_cache_breakpoint,
+)
 from litellm.responses.utils import auto_reasoning_summary
 from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
 from litellm.types.llms.anthropic import (
@@ -90,14 +94,28 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str
     return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
 
 
+def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]:
+    if "prompt_cache_breakpoint" not in block:
+        return cache_control_as_prompt_cache_breakpoint(block)
+    wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True)
+    return cache_control_as_prompt_cache_breakpoint(
+        with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker)
+    )
+
+
 def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None:
     """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints
     drops raw ones under drop_params."""
+    drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params"))
     if _reads_prompt_cache_breakpoint(model, forwarded_kwargs):
-        return cache_control_as_prompt_cache_breakpoint
+        return (
+            _cache_control_as_wire_safe_prompt_cache_breakpoint
+            if drops_params
+            else cache_control_as_prompt_cache_breakpoint
+        )
     if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs):
         return None
-    if drops_unsupported_params(forwarded_kwargs.get("drop_params")):
+    if drops_params:
         return _without_prompt_cache_breakpoint
     return None
 

litellm/llms/anthropic/prompt_cache_prediction.py

@@ -661,12 +661,13 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M
             TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()),
             dict(options),
         )
-        body: Final = AnthropicMessagesConfig().transform_anthropic_messages_request(
+        config: Final = AnthropicMessagesConfig()
+        body: Final = config.transform_anthropic_messages_request(
             model=resolved_model,
             messages=strip_provider_specific_fields_from_anthropic_messages(messages),
             anthropic_messages_optional_request_params=filtered,
             litellm_params=GenericLiteLLMParams(),
-            headers={},
+            headers=config.finalize_messages_headers({}, dict(filtered), list(messages)),  # pyright: ignore[reportUnknownMemberType]  # the shared helper takes and returns bare dicts
         )
         return MappingProxyType(_JSON_OBJECT.validate_python(body))
     except Exception:  # noqa: BLE001  # an unsupported hypothetical request is unavailable, never an inference failure

litellm/llms/azure/azure.py

@@ -33,6 +33,7 @@ from litellm.types.utils import (
 from litellm.utils import (
     CustomStreamWrapper,
     convert_to_model_response_object,
+    get_base_model_from_metadata,
 )
 
 from ...types.llms.openai import HttpxBinaryResponseContent
@@ -145,6 +146,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         azure_client: AzureOpenAI | OpenAI,
         data: dict,
         timeout: float | httpx.Timeout,
+        logging_obj: LiteLLMLoggingObj,
     ):
         """
         Helper to:
@@ -160,7 +162,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         except BadRequestError as e:
             if not is_output_token_limit_error(e):
                 raise
-            return build_output_token_limit_response(e=e, data=data, is_async=False)
+            return build_output_token_limit_response(
+                e=e,
+                data=data,
+                is_async=False,
+                base_model=get_base_model_from_metadata(logging_obj.model_call_details),
+            )
         except Exception as e:
             raise e
 
@@ -192,7 +199,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         except BadRequestError as e:
             if not is_output_token_limit_error(e):
                 raise
-            return build_output_token_limit_response(e=e, data=data, is_async=True)
+            return build_output_token_limit_response(
+                e=e,
+                data=data,
+                is_async=True,
+                base_model=get_base_model_from_metadata(logging_obj.model_call_details),
+            )
         except Exception as e:
             raise e
 
@@ -350,7 +362,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                     )
 
                 headers, response = self.make_sync_azure_openai_chat_completion_request(
-                    azure_client=azure_client, data=data, timeout=timeout
+                    azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj
                 )
                 if not isinstance(response, BaseModel):
                     raise AzureOpenAIError(
@@ -564,7 +576,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             },
         )
         headers, response = self.make_sync_azure_openai_chat_completion_request(
-            azure_client=azure_client, data=data, timeout=timeout
+            azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj
         )
         logging_obj.model_call_details["response_headers"] = headers
         streamwrapper: Final = CustomStreamWrapper(

litellm/llms/azure/chat/gpt_reasoning_transformation.py

@@ -24,6 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
 
     GPT5_SERIES_ROUTE = "gpt5_series/"
     OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none"))
+    LOOKS_UP_BASE_MODEL: ClassVar[bool] = True
 
     @classmethod
     def _model_map_lookup_name(cls, model: str) -> str:
@@ -88,7 +89,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
         drop_params: bool,
         api_version: str = "",
         *,
-        declared_reasoning_efforts: Collection[str] = (),
+        declared_reasoning_efforts: Collection[str] | None = None,
     ) -> dict:
         return OpenAIGPT5Config.map_openai_params(
             self,

litellm/llms/azure/chat/gpt_transformation.py

@@ -1,4 +1,4 @@
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
@@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_to_azure_openai_messages,
 )
+from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS
 from litellm.types.llms.azure import (
@@ -98,6 +99,22 @@ class AzureOpenAIConfig(BaseConfig):
             if key != "self" and value is not None:
                 setattr(self.__class__, key, value)
 
+    def translate_developer_role_to_system_role(
+        self,
+        messages: Sequence[AllMessageValues],
+        *,
+        model: str,
+        custom_llm_provider: str | None,
+        api_base: str | None,
+        model_info: object,
+    ) -> Sequence[AllMessageValues]:
+        """
+        Azure OpenAI accepts `system` messages at any position, so a developer message
+        only changes role. Declared here so `AzureOpenAIGPT5Config` resolves to it
+        ahead of the OpenAI-compatible hoist it also inherits.
+        """
+        return map_developer_role_to_system_role(messages=messages)
+
     @classmethod
     def get_config(cls):
         return super().get_config()

litellm/llms/azure/responses/transformation.py

@@ -335,7 +335,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         """
         Transform the get response API request into a URL and data
 
@@ -343,7 +343,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         - GET /v1/responses/{response_id}
         """
         get_url: Final = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id)
-        data: Final[dict] = {}
+        data: Final[dict[str, str]] = {}
         verbose_logger.debug("get response url=%s", get_url)
         return get_url, data
 

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -13,6 +13,7 @@ from pydantic import JsonValue
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.asyncify import asyncify
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.llms.anthropic.common_utils import AnthropicError
 from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
     AzureAIAnthropicCountTokensConfig,
@@ -69,6 +70,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
                 tools=tools,
                 system=system,
                 optional_params=optional_params,
+                drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")),
             )
 
             verbose_logger.debug("Transformed request: %s", request_body)

litellm/llms/azure_ai/anthropic/handler.py

@@ -4,11 +4,12 @@ Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authen
 
 import copy
 import json
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from typing import Final
 
 import httpx
 
+from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion
 from litellm.llms.anthropic.chat.transformation import JsonModeRequest
@@ -75,13 +76,22 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
             litellm_params=litellm_params,
         )
 
-        data: Final = config.transform_request(
+        request_body: Final[Mapping[str, object]] = config.transform_request(
             model=model,
             messages=messages,
             optional_params=optional_params,
             litellm_params=litellm_params,
             headers=headers,
         )
+        data: Final = dict(
+            strip_beta_gated_request_body(
+                request_body,
+                headers,  # pyright: ignore[reportUnknownArgumentType]  # validate_environment returns a bare dict
+                drop_params=litellm_params.get("drop_params"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # litellm_params is the handler's bare dict
+                model=model,
+                provider=custom_llm_provider,
+            )
+        )
 
         ## LOGGING
         logging_obj.pre_call(

litellm/llms/azure_ai/anthropic/messages_transformation.py

@@ -145,6 +145,8 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
             litellm_params=litellm_params,
             headers=headers,
         )
-        self._normalize_system_role_messages(anthropic_messages_request, model=model)
+        self._normalize_system_role_messages(
+            anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None)
+        )
         self._remove_scope_from_cache_control(anthropic_messages_request)
         return anthropic_messages_request

litellm/llms/azure_ai/anthropic/transformation.py

@@ -107,9 +107,11 @@ class AzureAnthropicConfig(AnthropicConfig):
             is_vertex_request=optional_params.get("is_vertex_request", False),
             user_anthropic_beta_headers=user_anthropic_beta_headers,
             mcp_server_used=mcp_server_used,
+            thinking=optional_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # optional_params is the base config's bare dict
         )
         # Merge headers - Azure auth (api-key or Authorization) takes precedence
-        headers = {**anthropic_headers, **headers}
+        merged_beta_header: Final = {k: v for k, v in anthropic_headers.items() if k == "anthropic-beta"}
+        headers = {**anthropic_headers, **headers, **merged_beta_header}
 
         # Ensure anthropic-version header is set
         if "anthropic-version" not in headers:

litellm/llms/azure_ai/chat/transformation.py

@@ -106,7 +106,7 @@ class AzureAIStudioConfig(OpenAIConfig):
         model: str,
         drop_params: bool,
         *,
-        declared_reasoning_efforts: Collection[str] = (),
+        declared_reasoning_efforts: Collection[str] | None = None,
     ) -> dict[str, object]:  # mutable-ok: OpenAIConfig.map_openai_params signature
         if not azureAIGPT5Config.is_model_gpt_5_model(model):
             return super().map_openai_params(

litellm/llms/azure_ai/passthrough/transformation.py

@@ -5,8 +5,9 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Final
 
 import httpx
-from pydantic import TypeAdapter, ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError
 
+from litellm._lazy_imports import get_default_encoding
 from litellm._logging import verbose_logger
 from litellm.llms.azure_ai.common_utils import (
     AzureFoundryModelInfo,
@@ -22,10 +23,15 @@ from litellm.llms.base_llm.passthrough.transformation import (
     relayed_body,
     strip_leading_model_segment,
 )
+from litellm.llms.cohere.embed.transformation import (
+    CohereEmbeddingRowShape,
+    CohereImageTokenBilling,
+    cohere_embedding_response,
+)
 from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_PASSTHROUGH_RESPONSE, NativeOcrPassthroughResponse
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.rerank import RerankResponse
-from litellm.types.utils import CallTypes, ImageResponse, StandardPassThroughResponseObject
+from litellm.types.utils import CallTypes, EmbeddingResponse, ImageResponse, StandardPassThroughResponseObject
 
 if TYPE_CHECKING:
     from httpx import URL, Response
@@ -35,6 +41,7 @@ if TYPE_CHECKING:
 
 
 EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({})
+ECHOED_TEXTS: Final = TypeAdapter(tuple[str, ...], config=ConfigDict(hide_input_in_errors=True))
 
 
 def api_version_from(litellm_params: Mapping[str, object]) -> str | None:
@@ -85,9 +92,29 @@ def relay_query_params(
     return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version})
 
 
+def echoed_texts(body: Mapping[str, object]) -> tuple[str, ...]:
+    try:
+        return ECHOED_TEXTS.validate_python(body.get("texts"))
+    except ValidationError:
+        return ()
+
+
+def cohere_embed_relay_response(body: Mapping[str, object]) -> EmbeddingResponse:
+    return cohere_embedding_response(
+        body,
+        None,
+        get_default_encoding(),
+        echoed_texts(body),
+        CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS,
+        CohereEmbeddingRowShape.TYPED,
+    )
+
+
 FOUNDRY_RELAY_SHAPES: Final = (
     RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate),
     RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate),
+    RelayShape("/providers/cohere/v1/embed", CallTypes.aembedding, cohere_embed_relay_response),
+    RelayShape("/providers/cohere/v2/embed", CallTypes.aembedding, cohere_embed_relay_response),
 )
 
 

litellm/llms/azure_ai/responses/transformation.py

@@ -232,7 +232,7 @@ class AzureAIAgentResponsesAPIConfig(AzureAIResponsesAPIConfig):
         model: str,
         drop_params: bool,
         *,
-        declared_reasoning_efforts: Collection[str],
+        declared_reasoning_efforts: Collection[str] | None,
     ) -> Mapping[str, object]:
         mapped: Final = super().map_openai_params_for_deployment(
             response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts

litellm/llms/base_llm/base_utils.py

@@ -10,14 +10,23 @@ from abc import ABC, abstractmethod
 from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
-from typing import Any, Final
+from typing import Any, Final, TypeAlias
 
 from openai.lib import _parsing, _pydantic  # pyright: ignore[reportPrivateUsage]  # SDK parser internals
 from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
+from typing_extensions import TypeIs  # noqa: TID251  # narrows a message to its system member by the role literal
 
 from litellm._logging import verbose_logger
+from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
-from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    ChatCompletionCachedContent,
+    ChatCompletionSystemMessage,
+    ChatCompletionTextObject,
+    ChatCompletionToolCallChunk,
+    ChatCompletionUserMessage,
+)
 from litellm.types.utils import (
     CountContextManagement,
     Message,
@@ -458,7 +467,7 @@ def type_to_response_format_param(
 
 
 def map_developer_role_to_system_role(
-    messages: list[AllMessageValues],
+    messages: Sequence[AllMessageValues],
 ) -> list[AllMessageValues]:
     """
     Translate `developer` role to `system` role for non-OpenAI providers.
@@ -473,3 +482,226 @@ def map_developer_role_to_system_role(
         else:
             new_messages.append(m)
     return new_messages
+
+
+SystemMessageContent: TypeAlias = str | list[object]
+
+_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None)
+
+
+def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent | None:
+    if "content" not in message:
+        return ""
+    try:
+        content: Final = _system_content_adapter.validate_python(message["content"])
+    except ValidationError:
+        return None
+    return "" if content is None else content
+
+
+def _message_cache_control(message: ChatCompletionSystemMessage) -> ChatCompletionCachedContent | None:
+    if "cache_control" not in message:
+        return None
+    return message["cache_control"]
+
+
+_SystemPart: TypeAlias = tuple[SystemMessageContent, ChatCompletionCachedContent | None]
+
+
+def _with_message_cache_control(
+    blocks: Sequence[object], cache_control: ChatCompletionCachedContent | None
+) -> tuple[object, ...]:
+    last: Final = as_str_mapping(blocks[-1]) if blocks else None
+    if last is None or cache_control is None or "cache_control" in last:
+        return tuple(blocks)
+    return (*blocks[:-1], {**last, "cache_control": cache_control})
+
+
+def _as_system_message(message: AllMessageValues) -> AllMessageValues:
+    if message["role"] != "developer":
+        return message
+    content_only: Final[ChatCompletionSystemMessage] = {"role": "system", "content": message["content"]}
+    named: Final[ChatCompletionSystemMessage] = (
+        content_only if "name" not in message else {**content_only, "name": message["name"]}
+    )
+    if "cache_control" not in message:
+        return named
+    cached: Final[ChatCompletionSystemMessage] = {**named, "cache_control": message["cache_control"]}
+    return cached
+
+
+def _text_blocks(
+    content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None
+) -> tuple[object, ...]:
+    if not isinstance(content, str):
+        return _with_message_cache_control(content, cache_control)
+    if not content:
+        return ()
+    if cache_control:
+        cached_block: Final[ChatCompletionTextObject] = {
+            "type": "text",
+            "text": content,
+            "cache_control": cache_control,
+        }
+        return (cached_block,)
+    text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": content}
+    return (text_block,)
+
+
+def _lone_text_block_text(blocks: Sequence[object]) -> str | None:
+    match blocks:
+        case [{"type": "text", "text": str() as text} as block] if block == {"type": "text", "text": text}:
+            return text
+        case _:
+            return None
+
+
+def _plain_text(content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None) -> str | None:
+    if cache_control:
+        return None
+    text: Final = content if isinstance(content, str) else _lone_text_block_text(content)
+    if text is None or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX):
+        return None
+    return text
+
+
+def _is_system_message(
+    message: AllMessageValues,
+) -> TypeIs[ChatCompletionSystemMessage]:  # guard-ok: the role literal picks the TypedDict member
+    return message["role"] == "system"
+
+
+def _with_leading_separator(blocks: tuple[object, ...]) -> tuple[object, ...]:
+    first: Final = as_str_mapping(blocks[0])
+    text: Final = None if first is None else first.get("text")
+    if first is None or not isinstance(text, str) or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX):
+        return blocks
+    return ({**first, "text": f"\n\n{text}"}, *blocks[1:])
+
+
+def _merged_system_content(parts: Sequence[_SystemPart]) -> SystemMessageContent:
+    texts: Final = tuple(itertools.starmap(_plain_text, parts))
+    if all(text is not None for text in texts):
+        return "\n\n".join(text for text in texts if text)
+    block_runs: Final = tuple(blocks for blocks in itertools.starmap(_text_blocks, parts) if blocks)
+    separated: Final = (*block_runs[:1], *map(_with_leading_separator, block_runs[1:]))
+    return list(itertools.chain.from_iterable(separated))
+
+
+def _merged_system_message(
+    run: Sequence[ChatCompletionSystemMessage], parts: Sequence[_SystemPart]
+) -> ChatCompletionSystemMessage:
+    content: Final = _merged_system_content(parts)
+    named: Final = next((message for message in reversed(run) if "name" in message), None)
+    if named is None:
+        merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content}
+        return merged
+    with_name: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content, "name": named["name"]}
+    return with_name
+
+
+def _merged_run(run: tuple[AllMessageValues, ...]) -> tuple[AllMessageValues, ...]:
+    system_run: Final = tuple(message for message in run if _is_system_message(message))
+    if len(system_run) < 2:
+        return run
+    contents: Final = tuple((_system_content(message), _message_cache_control(message)) for message in system_run)
+    parts: Final = tuple((content, cache_control) for content, cache_control in contents if content is not None)
+    if len(parts) != len(contents):
+        return run
+    return (_merged_system_message(system_run, parts),)
+
+
+def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMessageValues]:
+    runs: Final = (tuple(run) for _, run in itertools.groupby(messages, key=_is_system_message))
+    return itertools.chain.from_iterable(map(_merged_run, runs))
+
+
+_INSTRUCTION_ROLES: Final = frozenset(("system", "developer"))
+
+
+def _leading_instruction_block_length(roles: Sequence[str | None]) -> int:
+    return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles))
+
+
+def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int:
+    last_conversation_index: Final = next(
+        (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES),
+        None,
+    )
+    if last_conversation_index is None or roles[last_conversation_index] != "assistant":
+        return len(roles)
+    return last_conversation_index + 1
+
+
+def hoisted_instruction_indices(roles: Sequence[str | None]) -> tuple[int, ...]:
+    """
+    Indices of the leading instruction block plus every developer item after it, except
+    a developer block that closes the conversation right after an assistant turn, which
+    stays in place so the request does not end on the assistant's turn. Items without a
+    role (tool calls and their outputs) count as conversation.
+    """
+    leading_length: Final = _leading_instruction_block_length(roles)
+    closing_start: Final = _closing_instruction_block_start(roles, leading_length)
+    moved: Final = tuple(index for index in range(leading_length, closing_start) if roles[index] == "developer")
+    return (*range(leading_length), *moved)
+
+
+def closing_instruction_indices(roles: Sequence[str | None]) -> frozenset[int]:
+    closing_start: Final = _closing_instruction_block_start(roles, _leading_instruction_block_length(roles))
+    return frozenset(range(closing_start, len(roles)))
+
+
+def _role_name(role: object) -> str | None:
+    return role if isinstance(role, str) else None
+
+
+def _message_roles(messages: Sequence[AllMessageValues]) -> tuple[str | None, ...]:
+    return tuple(_role_name(message["role"]) for message in messages)
+
+
+def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, ...]:
+    hoisted: Final = hoisted_instruction_indices(_message_roles(messages))
+    hoisted_set: Final = frozenset(hoisted)
+    return (*hoisted, *(index for index in range(len(messages)) if index not in hoisted_set))
+
+
+def _developer_message_as_user(message: AllMessageValues) -> AllMessageValues:
+    if message["role"] != "developer":
+        return message
+    copied: Final = copy.deepcopy(message)
+    if "cache_control" not in copied:
+        as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": copied["content"]}
+        return as_user
+    cached_as_user: Final[ChatCompletionUserMessage] = {
+        "role": "user",
+        "content": copied["content"],
+        "cache_control": copied["cache_control"],
+    }
+    return cached_as_user
+
+
+def merge_system_message_runs(messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]:
+    """
+    Translate `developer` role to `system` role in place and fold each run of
+    consecutive system messages into one, for chat templates that render a single
+    system message per run.
+    """
+    return tuple(_merged_system_runs(tuple(map(_as_system_message, messages))))
+
+
+def hoist_developer_messages_into_leading_system_message(
+    messages: Sequence[AllMessageValues],
+) -> Sequence[AllMessageValues]:
+    """
+    Place instructions for an OpenAI-compatible model whose chat template allows a single
+    system message and only at the start: developer messages after the first user turn move
+    into the leading system block, a developer message that closes the conversation right
+    after an assistant turn becomes a user turn so the request does not end on the
+    assistant's turn, and each run of consecutive system messages folds into one.
+    """
+    order: Final = _hoisted_message_order(messages)
+    closing: Final = closing_instruction_indices(_message_roles(messages))
+    placed: Final = tuple(
+        _developer_message_as_user(messages[index]) if index in closing else messages[index] for index in order
+    )
+    return merge_system_message_runs(placed)

litellm/llms/base_llm/chat/transformation.py

@@ -4,7 +4,7 @@ Common base config for all LLM providers
 
 import types
 from abc import ABC, abstractmethod
-from collections.abc import AsyncIterator, Iterator, Mapping
+from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, Union
 
 import httpx
@@ -199,12 +199,19 @@ class BaseConfig(ABC):
 
     def translate_developer_role_to_system_role(
         self,
-        messages: list[AllMessageValues],
-    ) -> list[AllMessageValues]:
+        messages: Sequence[AllMessageValues],
+        *,
+        model: str,
+        custom_llm_provider: str | None,
+        api_base: str | None,
+        model_info: object,
+    ) -> Sequence[AllMessageValues]:
         """
         Translate `developer` role to `system` role for non-OpenAI providers.
 
-        Overriden by OpenAI/Azure
+        Overridden by OpenAI, which also places the instructions in one leading system
+        message for an OpenAI-compatible model declared to reject a later system
+        message, and by the o-series and Azure configs.
         """
         return map_developer_role_to_system_role(messages=messages)
 

litellm/llms/base_llm/responses/transformation.py

@@ -69,6 +69,9 @@ class BaseResponsesAPIConfig(ABC):
     def supports_encrypted_agent_messages(self) -> bool:
         return False
 
+    def supports_streamed_retrieval(self) -> bool:
+        return self.custom_llm_provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI)
+
     def sign_request(
         self,
         headers: dict,
@@ -209,7 +212,7 @@ class BaseResponsesAPIConfig(ABC):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         pass
 
     @abstractmethod
@@ -268,6 +271,9 @@ class BaseResponsesAPIConfig(ABC):
         """Returns True if litellm should fake a stream for the given model and stream value"""
         return False
 
+    def may_answer_stream_with_json(self) -> bool:
+        return False
+
     def supports_native_websocket(self) -> bool:
         """
         Returns True if the provider has a native WebSocket endpoint for Responses API.

litellm/llms/bedrock/chat/converse_transformation.py

@@ -17,6 +17,8 @@ from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.anthropic_beta_gated_fields import active_beta_values, log_beta_gated_strips, strip_beta_gated_thinking
+from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
 from litellm.constants import (
     BEDROCK_MIN_THINKING_BUDGET_TOKENS,
     RESPONSE_FORMAT_TOOL_NAME,
@@ -117,6 +119,7 @@ from litellm.utils import (
 )
 
 from ..common_utils import (
+    DROP_DISABLED_REASONING_EFFORT_WARNING,
     BedrockError,
     BedrockModelInfo,
     bedrock_cache_point_ttl_supported,
@@ -1192,11 +1195,7 @@ class AmazonConverseConfig(BaseConfig):
                         ),
                         status_code=400,
                     )
-                verbose_logger.debug(
-                    "Dropping unsupported `reasoning_effort=%s` for Bedrock model=%s.",
-                    value,
-                    model,
-                )
+                verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning_effort", value, model)
             elif param == "reasoning_effort" and isinstance(value, str):
                 self._handle_reasoning_effort_parameter(
                     model=model, reasoning_effort=value, optional_params=optional_params
@@ -1828,7 +1827,16 @@ class AmazonConverseConfig(BaseConfig):
                 filtered_tools.append(tool)
 
         base_model: Final = BedrockModelInfo.get_base_model(model)
-        client_beta_list: Final = get_anthropic_beta_from_headers(headers or {})
+        header_beta_list: Final = get_anthropic_beta_from_headers(headers or {})
+        client_beta_list: Final = header_beta_list or filter_and_transform_beta_headers(
+            sorted(
+                active_beta_values(
+                    {},
+                    additional_request_params.get("anthropic_beta"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # additional_request_params is a bare dict
+                )
+            ),
+            provider="bedrock_converse",
+        )
         eager_beta: Final = (
             (ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,)
             if base_model.startswith("anthropic")
@@ -1836,7 +1844,15 @@ class AmazonConverseConfig(BaseConfig):
             and ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER not in client_beta_list
             else ()
         )
-        anthropic_beta_list: Final = [*client_beta_list, *eager_beta]
+        thinking_betas: Final = AnthropicModelInfo().thinking_betas(
+            additional_request_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # additional_request_params is a bare dict
+        )
+        missing_thinking_betas: Final = (
+            tuple(beta for beta in thinking_betas if beta not in client_beta_list)
+            if base_model.startswith("anthropic")
+            else ()
+        )
+        anthropic_beta_list: Final = [*client_beta_list, *eager_beta, *missing_thinking_betas]
 
         # Only separate tools if computer use tools are actually present
         if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model):
@@ -1906,7 +1922,8 @@ class AmazonConverseConfig(BaseConfig):
                 else:
                     computer_use_header = "computer-use-2024-10-22"
 
-                anthropic_beta_list.append(computer_use_header)
+                if computer_use_header not in anthropic_beta_list:
+                    anthropic_beta_list.append(computer_use_header)
                 # Transform computer use tools to proper Bedrock format
                 transformed_computer_tools: Final = self._transform_computer_use_tools(computer_use_tools)
                 additional_request_params["tools"] = transformed_computer_tools
@@ -2031,6 +2048,16 @@ class AmazonConverseConfig(BaseConfig):
         bedrock_tools, anthropic_beta_list = self._process_tools_and_beta(
             original_tools, model, headers, additional_request_params, litellm_params, deployment_model
         )
+        requested_thinking: Final = additional_request_params.get("thinking")  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # _prepare_request_params hands back a bare dict
+        body_betas: Final = additional_request_params.get("anthropic_beta")  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # same bare dict
+        gated_thinking: Final = strip_beta_gated_thinking(
+            requested_thinking,  # pyright: ignore[reportUnknownArgumentType]  # same bare dict
+            active_beta_values({}, body_betas) | frozenset(anthropic_beta_list),  # pyright: ignore[reportUnknownArgumentType]  # _process_tools_and_beta hands back a bare list
+            drop_params=drop_params,
+        )
+        if gated_thinking.stripped:
+            log_beta_gated_strips(gated_thinking.stripped, model=model, provider="bedrock_converse")
+            additional_request_params["thinking"] = gated_thinking.thinking
 
         # Append cachePoint to tools if cache_control_injection_points has tool_config
         cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None)

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
 import httpx
 from pydantic import TypeAdapter
 
+from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body
 from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
 from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -210,7 +211,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         if beta_list:
             _anthropic_request["anthropic_beta"] = beta_list
 
-        return _anthropic_request
+        return dict(
+            strip_beta_gated_request_body(
+                _anthropic_request,  # pyright: ignore[reportUnknownArgumentType]  # built by the base signature's bare dict transform
+                {},
+                drop_params=litellm_params.get("drop_params"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # litellm_params is the base signature's bare dict
+                model=model,
+                provider="bedrock_invoke",
+            )
+        )
 
     @property
     def uses_async_transform_request(self) -> bool:
@@ -294,7 +303,6 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         programmatic_tool_calling_used: Final = self.is_programmatic_tool_calling_used(tools)
         input_examples_used: Final = self.is_input_examples_used(tools)
         is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages)
-        is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking)
 
         client_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
         user_beta_set: Final = (
@@ -312,7 +320,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
             mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")),
             custom_llm_provider="bedrock",
             is_mid_conversation_output_config_used=is_mid_conversation_output_config_used,
-            is_thinking_display_updates_used=is_thinking_display_updates_used,
+            thinking=thinking,
         )
         beta_set.update(auto_betas)
 

litellm/llms/bedrock/claude_platform/transformation.py

@@ -72,6 +72,7 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig):
             ),
             code_execution_tool_used=self.is_code_execution_tool_used(tools=optional_params.get("tools")),
             container_with_skills_used=self.is_container_with_skills_used(optional_params=optional_params),
+            thinking=optional_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # optional_params is the base config's bare dict
         )
         anthropic_headers["anthropic-workspace-id"] = workspace_id
         return {**headers, **anthropic_headers}

litellm/llms/bedrock/common_utils.py

@@ -15,7 +15,11 @@ from dataclasses import dataclass
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
 
-from typing_extensions import ReadOnly, TypedDict
+from typing_extensions import (
+    ReadOnly,
+    TypedDict,
+    TypeIs,  # noqa: TID251  # narrows the caller's tools without copying them
+)
 
 if TYPE_CHECKING:
     from botocore.eventstream import EventStreamMessage
@@ -573,6 +577,26 @@ def strip_unsupported_bedrock_invoke_output_config_keys(
         request_body["output_config"] = {"format": preserved_format}  # rebind-ok: out-param
 
 
+_TOOL_ITEMS_ADAPTER: Final = TypeAdapter(tuple[object, ...])
+
+
+def _is_str_keyed_dict(
+    value: object,
+) -> TypeIs[dict[str, object]]:  # guard-ok: JSON tool objects  # mutable-ok: fallbacks reuse tools edited in place
+    return isinstance(value, dict)
+
+
+def _normalize_custom_field_on_tool(tool: object) -> None:
+    if not _is_str_keyed_dict(tool):
+        return
+    custom: Final = tool.pop("custom", None)
+    if not _is_str_keyed_dict(custom) or "defer_loading" in tool:
+        return
+    deferred: Final = custom.get("defer_loading")
+    if isinstance(deferred, bool):
+        tool["defer_loading"] = deferred  # rebind-ok: a fallback reuses the caller's tool, as on main
+
+
 def normalize_custom_field_on_tools(request_body: dict) -> None:
     """
     Drop the ``custom`` field from each tool, first hoisting a boolean
@@ -590,15 +614,8 @@ def normalize_custom_field_on_tools(request_body: dict) -> None:
     tools: Final = request_body.get("tools")
     if not tools or not isinstance(tools, list):
         return
-    for tool in tools:
-        if not isinstance(tool, dict):
-            continue
-        custom: dict[str, object] | None = tool.pop("custom", None)
-        if not isinstance(custom, dict) or "defer_loading" in tool:
-            continue
-        deferred: object = custom.get("defer_loading")
-        if isinstance(deferred, bool):
-            tool["defer_loading"] = deferred
+    for tool in _TOOL_ITEMS_ADAPTER.validate_python(tools):
+        _normalize_custom_field_on_tool(tool)
 
 
 _TOOL_DICTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
@@ -1314,6 +1331,12 @@ def _mantle_api_base_from_env() -> str | None:
     return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base)
 
 
+DROP_DISABLED_REASONING_EFFORT_WARNING: Final = (
+    "Dropping %s=%r for Bedrock model=%r (drop_params=True) because the model map marks that level unsupported. "
+    "Send a level the model supports."
+)
+
+
 def bedrock_reasoning_effort_disabled(model: str, effort: str) -> bool:
     from litellm.utils import is_explicitly_disabled_factory
 

litellm/llms/bedrock/count_tokens/mantle_handler.py

@@ -8,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.anthropic.common_utils import tools_without_custom_field
 from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
 from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url
@@ -26,6 +27,7 @@ class MantleCountTokensRequest(TypedDict):
 _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest)
 _COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue])
 _COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue])
+_TOOLS: Final = TypeAdapter(list[dict[str, JsonValue]])
 
 
 class BedrockMantleCountTokensHandler(BaseAWSLLM):
@@ -59,10 +61,15 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
                 self.get_aws_region_name, optional_params=litellm_params, model=resolved_model, model_id=None
             )
             endpoint_url: Final = await run_aws_signing(self.get_mantle_count_tokens_endpoint, aws_region_name)
+            tools: Final = request.get("tools")
             body: Final = self._anthropic_config.transform_request_to_count_tokens(
                 model=resolved_model,
                 messages=request["messages"],
-                tools=request.get("tools"),
+                tools=None
+                if tools is None
+                else _TOOLS.validate_python(
+                    tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True)
+                ),
                 system=request.get("system"),
                 optional_params=_COUNT_OPTIONS.validate_python(
                     {name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES}

litellm/llms/bedrock/count_tokens/transformation.py

@@ -12,6 +12,7 @@ from typing import Final, Literal
 from pydantic import JsonValue
 
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.anthropic.common_utils import tools_without_custom_field
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.bedrock.common_utils import get_bedrock_base_model
 
@@ -194,7 +195,15 @@ class BedrockCountTokensConfig(BaseAWSLLM):
 
         # For InvokeModel, we need to provide the raw body that would be sent to the model
         # Remove the 'model' field from the body as it's not part of the model input
-        body_data: Final = {k: v for k, v in request_data.items() if k != "model"}
+        tools: Final = request_data.get("tools")
+        body_data: Final = {
+            **{k: v for k, v in request_data.items() if k != "model"},
+            **(
+                {"tools": tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True)}
+                if isinstance(tools, list)
+                else {}
+            ),
+        }
 
         if "messages" in body_data:
             # Bedrock validates the body against the model's InvokeModel schema;

litellm/llms/bedrock/embed/embedding.py

@@ -9,9 +9,11 @@ from collections.abc import Callable, Mapping
 from typing import TYPE_CHECKING, Final, get_args, overload
 
 import httpx
+from pydantic import TypeAdapter
 
 import litellm
 from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
+from litellm.litellm_core_utils.hidden_params import set_hidden_param
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.cohere.embed.handler import embedding as cohere_embedding
 from litellm.llms.custom_httpx.http_handler import (
@@ -36,7 +38,7 @@ from ..base_aws_llm import (
     pop_aws_auth_params,
     run_aws_signing,
 )
-from ..common_utils import BedrockError
+from ..common_utils import BedrockError, split_bedrock_region_path
 from .amazon_nova_transformation import AmazonNovaEmbeddingConfig
 from .amazon_titan_g1_transformation import AmazonTitanG1Config
 from .amazon_titan_multimodal_transformation import (
@@ -64,12 +66,21 @@ def _sign_get_request(
     return request.prepare()
 
 
+_OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
+
+
+def _without_region_path(model: str) -> str:
+    region, model_id = split_bedrock_region_path(model)
+    return model if region is None else model_id
+
+
 class BedrockEmbedding(BaseAWSLLM):
     @overload
     def _load_credentials(
         self,
         optional_params: dict,  # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
         bearer_token: None = None,
+        region_from_model: str | None = None,
     ) -> tuple[Credentials, str]: ...
 
     @overload
@@ -77,15 +88,18 @@ class BedrockEmbedding(BaseAWSLLM):
         self,
         optional_params: dict,  # mutable-ok: the implementation pops the aws_* keys out of the caller's dict in place
         bearer_token: str,
+        region_from_model: str | None = None,
     ) -> tuple[None, str]: ...
 
     def _load_credentials(
         self,
         optional_params: dict,
         bearer_token: str | None = None,
+        region_from_model: str | None = None,
     ) -> tuple[Credentials | None, str]:
         auth_params: Final = pop_aws_auth_params(optional_params)
-        aws_region_name = optional_params.pop("aws_region_name", None)
+        explicit_region_name: Final = optional_params.pop("aws_region_name", None)
+        aws_region_name = region_from_model if explicit_region_name is None else explicit_region_name
 
         ### SET REGION NAME ###
         if aws_region_name is None:
@@ -184,14 +198,17 @@ class BedrockEmbedding(BaseAWSLLM):
         provider: BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
         is_async_invoke: bool | None = False,
         batch_data: list[dict] | None = None,
+        aws_region_name: str | None = None,
     ) -> EmbeddingResponse | None:
         """
         Transforms the response from the Bedrock embedding provider to the OpenAI format.
         """
         returned_response: EmbeddingResponse | None = None
+        bedrock_model: Final = _without_region_path(model)
+        is_async_job: Final = bool(is_async_invoke) and len(response_list) == 1 and "invocationArn" in response_list[0]
 
         # Handle async invoke responses (single response with invocationArn)
-        if is_async_invoke and len(response_list) == 1 and "invocationArn" in response_list[0]:
+        if is_async_job:
             if provider == "twelvelabs":
                 returned_response = TwelveLabsMarengoEmbeddingConfig().transform_async_invoke_response(
                     response=response_list[0], model=model
@@ -227,15 +244,15 @@ class BedrockEmbedding(BaseAWSLLM):
                 )
         else:
             # Handle regular invoke responses
-            if model == "amazon.titan-embed-image-v1":
+            if bedrock_model == "amazon.titan-embed-image-v1":
                 returned_response = AmazonTitanMultimodalEmbeddingG1Config().transform_response(
                     response_list=response_list, model=model, batch_data=batch_data
                 )
-            elif model == "amazon.titan-embed-text-v1":
+            elif bedrock_model == "amazon.titan-embed-text-v1":
                 returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model)
-            elif model == "amazon.titan-embed-text-v2:0":
+            elif bedrock_model == "amazon.titan-embed-text-v2:0":
                 returned_response = AmazonTitanV2Config().transform_response(response_list=response_list, model=model)
-            elif model == "amazon.titan-embed-g1-text-02":
+            elif bedrock_model == "amazon.titan-embed-g1-text-02":
                 returned_response = AmazonTitanG1Config().transform_response(response_list=response_list, model=model)
             elif provider == "twelvelabs":
                 returned_response = TwelveLabsMarengoEmbeddingConfig().transform_response(
@@ -251,6 +268,8 @@ class BedrockEmbedding(BaseAWSLLM):
         ##########################################################
         if returned_response is None:
             raise Exception(f"Unable to map model response to known provider format. model={model}")
+        if aws_region_name is not None and not is_async_job:
+            set_hidden_param(returned_response, "region_name", aws_region_name)
         return returned_response
 
     def _single_func_embeddings(
@@ -319,6 +338,7 @@ class BedrockEmbedding(BaseAWSLLM):
             provider=provider,
             is_async_invoke=is_async_invoke,
             batch_data=batch_data,
+            aws_region_name=aws_region_name,
         )
 
     async def _async_single_func_embeddings(
@@ -390,6 +410,7 @@ class BedrockEmbedding(BaseAWSLLM):
             provider=provider,
             is_async_invoke=is_async_invoke,
             batch_data=batch_data,
+            aws_region_name=aws_region_name,
         )
 
     def embeddings(
@@ -409,26 +430,29 @@ class BedrockEmbedding(BaseAWSLLM):
         litellm_params: dict,
         api_key: str | None = None,
     ) -> EmbeddingResponse:
+        model_id_param: Final = _OPTIONAL_STR.validate_python(optional_params.pop("model_id", None)) or None
+        has_async_invoke: Final = "async_invoke/" in model
+        model_without_async_route: Final = model.replace("async_invoke/", "", 1)
+        path_region: Final = split_bedrock_region_path(model_without_async_route)[0] if model_id_param is None else None
+        bedrock_model: Final = _without_region_path(model_without_async_route)
         credentials, aws_region_name = self._load_credentials(
-            optional_params, bearer_token=bedrock_bearer_token(api_key)
+            optional_params,
+            bearer_token=bedrock_bearer_token(api_key),
+            region_from_model=self._get_aws_region_from_model_arn(model_id_param) or path_region,
         )
 
         ### TRANSFORMATION ###
-        unencoded_model_id: Final = optional_params.pop("model_id", None) or model  # default to model if not passed
+        unencoded_model_id: Final = model_id_param or bedrock_model
         modelId: Final = urllib.parse.quote(unencoded_model_id, safe="")
         aws_region_name = self._get_aws_region_name(
             optional_params={"aws_region_name": aws_region_name},
-            model=model,
+            model=bedrock_model,
             model_id=unencoded_model_id,
         )
-        # Check async invoke needs to be used
-        has_async_invoke: Final = "async_invoke/" in model
-        if has_async_invoke:
-            model = model.replace("async_invoke/", "", 1)
-        provider: Final = self.get_bedrock_embedding_provider(model)
+        provider: Final = self.get_bedrock_embedding_provider(bedrock_model)
         if provider is None:
             raise Exception(
-                f"Unable to determine bedrock embedding provider for model: {model}. "
+                f"Unable to determine bedrock embedding provider for model: {bedrock_model}. "
                 f"Supported providers: {list(get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL))}"
             )
         inference_params = copy.deepcopy(optional_params)
@@ -441,9 +465,9 @@ class BedrockEmbedding(BaseAWSLLM):
         batch_data: list | None = None
         if provider == "cohere":
             data = BedrockCohereEmbeddingConfig().transform_request(
-                model=model, input=input, inference_params=inference_params
+                model=bedrock_model, input=input, inference_params=inference_params
             )
-        elif provider == "amazon" and model in [
+        elif provider == "amazon" and bedrock_model in [
             "amazon.titan-embed-image-v1",
             "amazon.titan-embed-text-v1",
             "amazon.titan-embed-text-v2:0",
@@ -451,28 +475,28 @@ class BedrockEmbedding(BaseAWSLLM):
         ]:
             batch_data = []
             for i in input:
-                if model == "amazon.titan-embed-image-v1":
+                if bedrock_model == "amazon.titan-embed-image-v1":
                     transformed_request: AmazonEmbeddingRequest = (
                         AmazonTitanMultimodalEmbeddingG1Config().transform_request(
                             input=i, inference_params=inference_params
                         )
                     )
-                elif model == "amazon.titan-embed-text-v1":
+                elif bedrock_model == "amazon.titan-embed-text-v1":
                     transformed_request = AmazonTitanG1Config().transform_request(
                         input=i, inference_params=inference_params
                     )
-                elif model == "amazon.titan-embed-text-v2:0":
+                elif bedrock_model == "amazon.titan-embed-text-v2:0":
                     transformed_request = AmazonTitanV2Config().transform_request(
                         input=i, inference_params=inference_params
                     )
-                elif model == "amazon.titan-embed-g1-text-02":
+                elif bedrock_model == "amazon.titan-embed-g1-text-02":
                     transformed_request = AmazonTitanG1Config().transform_request(
                         input=i, inference_params=inference_params
                     )
                 else:
                     raise Exception(
                         "Unmapped model. Received={}. Expected={}".format(
-                            model,
+                            bedrock_model,
                             [
                                 "amazon.titan-embed-image-v1",
                                 "amazon.titan-embed-text-v1",
@@ -485,7 +509,7 @@ class BedrockEmbedding(BaseAWSLLM):
         elif provider == "twelvelabs":
             batch_data = []
             for i in input:
-                twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=model).transform_request(
+                twelvelabs_request = TwelveLabsMarengoEmbeddingConfig(model=bedrock_model).transform_request(
                     input=i,
                     inference_params=inference_params,
                     async_invoke_route=has_async_invoke,
@@ -527,7 +551,7 @@ class BedrockEmbedding(BaseAWSLLM):
                     extra_headers=extra_headers,
                     endpoint_url=endpoint_url,
                     aws_region_name=aws_region_name,
-                    model=model,
+                    model=model_without_async_route,
                     logging_obj=logging_obj,
                     api_key=api_key,
                     provider=provider,
@@ -541,7 +565,7 @@ class BedrockEmbedding(BaseAWSLLM):
                 extra_headers=extra_headers,
                 endpoint_url=endpoint_url,
                 aws_region_name=aws_region_name,
-                model=model,
+                model=model_without_async_route,
                 logging_obj=logging_obj,
                 api_key=api_key,
                 provider=provider,
@@ -570,8 +594,9 @@ class BedrockEmbedding(BaseAWSLLM):
         ## ROUTING ##
         # Convert CaseInsensitiveDict to regular dict for httpx compatibility
         headers_for_request: Final = dict(prepped.headers) if hasattr(prepped, "headers") else {}
+        set_hidden_param(model_response, "region_name", aws_region_name)
         return cohere_embedding(
-            model=model,
+            model=model_without_async_route,
             input=input,
             model_response=model_response,
             logging_obj=logging_obj,

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py

@@ -7,6 +7,7 @@ from pydantic import TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_native_request
 from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
 from litellm.constants import (
     BEDROCK_MIN_THINKING_BUDGET_TOKENS,
@@ -102,6 +103,9 @@ class AmazonAnthropicClaudeMessagesConfig(
     def should_strip_billing_metadata(self) -> bool:
         return True
 
+    def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]:
+        return tools
+
     def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
         return True
 
@@ -546,6 +550,9 @@ class AmazonAnthropicClaudeMessagesConfig(
             "edits": retained_edits,
         }
 
+    def strips_beta_gated_fields_from_final_headers(self) -> bool:
+        return False
+
     def _get_bedrock_invoke_anthropic_beta_headers(
         self,
         model: str,
@@ -581,9 +588,7 @@ class AmazonAnthropicClaudeMessagesConfig(
             ),
             custom_llm_provider="bedrock",
             is_mid_conversation_output_config_used=is_mid_conversation_output_config_used,
-            is_thinking_display_updates_used=anthropic_model_info.is_thinking_display_updates_used(
-                anthropic_messages_request.get("thinking")
-            ),
+            thinking=anthropic_messages_request.get("thinking"),
             is_mid_conversation_tool_change_used=anthropic_model_info.is_mid_conversation_tool_change_used(
                 outgoing_messages_typed
             ),
@@ -754,7 +759,9 @@ class AmazonAnthropicClaudeMessagesConfig(
                 ),
                 "display": "updates",
             }
-        self._normalize_system_role_messages(anthropic_messages_request, model=model)
+        self._normalize_system_role_messages(
+            anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None)
+        )
         #########################################################
         ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ###
         #########################################################
@@ -822,6 +829,20 @@ class AmazonAnthropicClaudeMessagesConfig(
         if filtered_betas:
             anthropic_messages_request["anthropic_beta"] = filtered_betas
 
+        final_request: Final = OBJECT_MAPPING_ADAPTER.validate_python(anthropic_messages_request)  # pyright: ignore[reportUnknownArgumentType]  # built from the base signature's bare dict
+        gated: Final = strip_beta_gated_native_request(
+            {},
+            final_request,
+            final_request.get("messages"),
+            body_betas=filtered_betas,
+            drop_params=getattr(litellm_params, "drop_params", None),
+        )
+        log_beta_gated_strips(gated.stripped, model=model, provider="bedrock_invoke")
+        if gated.thinking_stripped:
+            anthropic_messages_request["thinking"] = gated.thinking
+        if gated.messages_stripped:
+            anthropic_messages_request["messages"] = list(gated.messages)
+
         outbound_tools: Final = tools_without_eager_input_streaming(anthropic_messages_request)
         if outbound_tools is not None:
             anthropic_messages_request["tools"] = outbound_tools

litellm/llms/bedrock/responses/transformation.py

@@ -52,6 +52,7 @@ from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools,
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.bedrock.common_utils import (
     BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX,
+    DROP_DISABLED_REASONING_EFFORT_WARNING,
     BedrockError,
     bedrock_reasoning_effort_disabled,
     bedrock_supports_openai_responses,
@@ -173,7 +174,7 @@ def _without_disabled_reasoning_effort(
             ),
             status_code=400,
         )
-    verbose_logger.debug("Dropping unsupported `reasoning.effort=%s` for Bedrock model=%s.", effort, model)
+    verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning.effort", effort, model)
     rest: Final = {key: value for key, value in reasoning.items() if key != "effort"}
     without_reasoning: Final = {key: value for key, value in params.items() if key != "reasoning"}
     return {**without_reasoning, "reasoning": rest} if rest else without_reasoning

litellm/llms/bigmodel/chat/transformation.py

@@ -0,0 +1,59 @@
+from collections.abc import Mapping
+from typing import Final
+
+from litellm.llms.zai.chat.transformation import ZAIChatConfig
+
+from ..common_utils import get_bigmodel_api_key, get_bigmodel_url, is_unsupported_disabled_thinking
+
+_SUPPORTED_PARAMS: Final = (
+    "max_tokens",
+    "max_completion_tokens",
+    "stream",
+    "stream_options",
+    "temperature",
+    "top_p",
+    "stop",
+    "tools",
+    "tool_choice",
+    "response_format",
+    "thinking",
+    "reasoning_effort",
+)
+
+
+class BigModelChatConfig(ZAIChatConfig):
+    @property
+    def custom_llm_provider(self) -> str:
+        return "bigmodel"
+
+    def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
+        return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
+
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: inherited interface requires a list
+        return list(_SUPPORTED_PARAMS)
+
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: request pipeline adds parameters to the returned body
+        mapped: Final = {
+            **optional_params,
+            **{
+                ("max_tokens" if key == "max_completion_tokens" else key): value
+                for key, value in non_default_params.items()
+                if key in _SUPPORTED_PARAMS and key != "thinking"
+            },
+        }
+        if "thinking" not in non_default_params:
+            return mapped
+        thinking: Final = non_default_params["thinking"]
+        if is_unsupported_disabled_thinking(model, thinking):
+            return {
+                **mapped,
+                "reasoning_effort": mapped.get("reasoning_effort", "low"),
+                "extra_body": {"thinking": {"type": "enabled"}},
+            }
+        return {**mapped, "extra_body": {"thinking": thinking}}

litellm/llms/bigmodel/common_utils.py

@@ -0,0 +1,92 @@
+import re
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final, Literal
+from urllib.parse import urlsplit, urlunsplit
+
+import httpx
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+import litellm
+from litellm.exceptions import ContentPolicyViolationError, ContextWindowExceededError
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+
+BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn"
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_BUSINESS_CODE_IN_ERROR: Final = re.compile(r"""['"]code['"]\s*:\s*['"]?(\d{4})\b""")
+_CODING_CHAT_PATH: Final = "/api/coding/paas/v4"
+_API_PATHS: Final = (
+    "/api/coding/paas/v4/chat/completions",
+    _CODING_CHAT_PATH,
+    "/api/paas/v4/chat/completions",
+    "/api/anthropic/v1/messages",
+    "/api/anthropic/v1",
+    "/api/v1/responses",
+    "/api/paas/v4",
+    "/api/anthropic",
+    "/api/v1",
+    "/api",
+)
+_ENDPOINT_PATHS: Final = MappingProxyType(
+    {
+        "chat": "/api/paas/v4",
+        "responses": "/api/v1/responses",
+        "messages": "/api/anthropic/v1/messages",
+    }
+)
+
+
+def get_bigmodel_api_key(api_key: str | None) -> str | None:
+    return api_key or get_secret_str("BIGMODEL_API_KEY")
+
+
+def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses", "messages"]) -> str:
+    resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE
+    parsed: Final = urlsplit(resolved_base)
+    path: Final = parsed.path.rstrip("/")
+    suffix: Final = next((candidate for candidate in _API_PATHS if path.endswith(candidate)), "")
+    endpoint_path: Final = (
+        _CODING_CHAT_PATH if endpoint == "chat" and suffix.startswith(_CODING_CHAT_PATH) else _ENDPOINT_PATHS[endpoint]
+    )
+    return urlunsplit(
+        (parsed.scheme, parsed.netloc, f"{path.removesuffix(suffix)}{endpoint_path}", parsed.query, parsed.fragment)
+    )
+
+
+def bigmodel_json_body(response: httpx.Response) -> Mapping[str, JsonValue]:
+    try:
+        return _JSON_OBJECT.validate_json(response.content)
+    except ValidationError:
+        raise BaseLLMException(message=response.text, status_code=response.status_code, headers=response.headers)
+
+
+def is_unsupported_disabled_thinking(model: str, thinking: object) -> bool:
+    return thinking == {"type": "disabled"} and litellm.utils.supports_factory(
+        model=model, custom_llm_provider="bigmodel", key="thinking_always_on"
+    )
+
+
+def bigmodel_business_error(
+    code: JsonValue, message: str, model: str, response: httpx.Response | None
+) -> ContextWindowExceededError | ContentPolicyViolationError | None:
+    match code:
+        case 1261 | "1261":
+            return ContextWindowExceededError(message=message, model=model, llm_provider="bigmodel", response=response)
+        case 1301 | "1301":
+            return ContentPolicyViolationError(message=message, model=model, llm_provider="bigmodel", response=response)
+        case _:
+            return None
+
+
+def raise_bigmodel_business_error(model: object, error_str: str, original_exception: object) -> None:
+    found: Final = _BUSINESS_CODE_IN_ERROR.search(error_str)
+    response: Final = getattr(original_exception, "response", None)
+    error: Final = bigmodel_business_error(
+        found.group(1) if found else None,
+        error_str,
+        str(model),
+        response if isinstance(response, httpx.Response) else None,
+    )
+    if error is not None:
+        raise error

litellm/llms/bigmodel/messages/transformation.py

@@ -0,0 +1,206 @@
+import json
+from collections.abc import AsyncIterator, Mapping, Sequence
+from typing import Final
+
+import httpx
+from openai._streaming import ServerSentEvent, SSEDecoder
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+from litellm.exceptions import AuthenticationError
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.llms.openai_like.messages.transformation import (
+    DEFAULT_ANTHROPIC_API_VERSION,
+    OpenAILikeAnthropicMessagesConfig,
+)
+from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
+from litellm.types.router import GenericLiteLLMParams
+
+from ..common_utils import (
+    bigmodel_json_body,
+    get_bigmodel_api_key,
+    get_bigmodel_url,
+    is_unsupported_disabled_thinking,
+)
+
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_OUTPUT_CONFIG: Final = TypeAdapter(dict[str, object])
+
+
+def _with_lowest_effort_thinking(
+    params: Mapping[str, object],
+) -> dict[str, object]:  # mutable-ok: the parent transform takes a request dict
+    requested_output_config: Final = params.get("output_config")
+    output_config: Final = {
+        **({} if "reasoning_effort" in params else {"effort": "low"}),
+        **(
+            _OUTPUT_CONFIG.validate_python(requested_output_config) if isinstance(requested_output_config, dict) else {}
+        ),
+    }
+    return {**params, "thinking": {"type": "adaptive"}, **({"output_config": output_config} if output_config else {})}
+
+
+def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]:  # mutable-ok: native JSON response
+    usage: Final = body.get("usage")
+    if not isinstance(usage, dict):
+        return dict(body)
+    server_tools: Final = usage.get("server_tool_use")
+    if not isinstance(server_tools, dict) or "web_fetch_requests" in server_tools:
+        return dict(body)
+    normalized: Final[JsonValue] = {
+        **body,
+        "usage": {**usage, "server_tool_use": {"web_fetch_requests": 0, **server_tools}},
+    }
+    return normalized
+
+
+class _BigModelMessagesStream(httpx.AsyncByteStream):
+    def __init__(self, response: httpx.Response) -> None:
+        self._response = response
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        async for event in SSEDecoder().aiter_bytes(self._response.aiter_bytes()):
+            yield _normalize_event(event)
+
+    async def aclose(self) -> None:
+        await self._response.aclose()
+
+
+def _json_object(data: str) -> Mapping[str, JsonValue] | None:
+    try:
+        return _JSON_OBJECT.validate_json(data)
+    except ValidationError:
+        return None
+
+
+def _normalized_data(data: str) -> str:
+    body: Final = _json_object(data)
+    if body is None:
+        return data
+    message: Final = body.get("message")
+    normalized: Final[JsonValue] = (
+        {**body, "message": _normalize_message(message)}
+        if body.get("type") == "message_start" and isinstance(message, dict)
+        else _normalize_message(body)
+    )
+    return data if normalized == body else json.dumps(normalized, ensure_ascii=False)
+
+
+def _normalize_event(event: ServerSentEvent) -> bytes:
+    metadata: Final = "".join(
+        f"{name}: {value}\n"
+        for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry))
+        if value is not None
+    )
+    data_lines: Final = "".join(f"data: {line}\n" for line in _normalized_data(event.data).split("\n"))
+    return f"{metadata}{data_lines}\n".encode()
+
+
+class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig):
+    @property
+    def custom_llm_provider(self) -> str:
+        return "bigmodel"
+
+    def is_native_messages_route(self) -> bool:
+        return True
+
+    def should_strip_billing_metadata(self) -> bool:
+        return True
+
+    def transform_anthropic_messages_request(
+        self,
+        model: str,
+        messages: list[dict[str, object]],  # mutable-ok: matches dict-typed base signature
+        anthropic_messages_optional_request_params: dict[str, object],  # mutable-ok: matches dict-typed base signature
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: matches dict-typed base signature
+    ) -> dict[str, object]:  # mutable-ok: matches dict-typed base signature
+        request_params: Final = (
+            _with_lowest_effort_thinking(anthropic_messages_optional_request_params)
+            if is_unsupported_disabled_thinking(model, anthropic_messages_optional_request_params.get("thinking"))
+            else anthropic_messages_optional_request_params
+        )
+        return super().transform_anthropic_messages_request(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the parent transform returns a bare dict
+            model, messages, request_params, litellm_params, headers
+        )
+
+    def transform_anthropic_messages_response(
+        self, model: str, raw_response: httpx.Response, logging_obj: Logging
+    ) -> AnthropicMessagesResponse:
+        normalized: Final = httpx.Response(
+            raw_response.status_code,
+            json=_normalize_message(bigmodel_json_body(raw_response)),
+            request=raw_response.request,
+        )
+        return super().transform_anthropic_messages_response(model, normalized, logging_obj)
+
+    def get_async_streaming_response_iterator(
+        self,
+        model: str,
+        httpx_response: httpx.Response,
+        request_body: dict[str, object],  # mutable-ok: inherited native stream API takes a request dict
+        litellm_logging_obj: Logging,
+    ) -> AsyncIterator[bytes]:
+        normalized: Final = httpx.Response(
+            httpx_response.status_code,
+            headers={
+                key: value
+                for key, value in httpx_response.headers.items()
+                if key.lower() not in ("content-encoding", "content-length")
+            },
+            stream=_BigModelMessagesStream(httpx_response),
+            request=httpx_response.request,
+        )
+        return super().get_async_streaming_response_iterator(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # native parent yields bytes but its iterator annotation omits the element type
+            model, normalized, request_body, litellm_logging_obj
+        )
+
+    @staticmethod
+    def _translate_reasoning_effort_to_anthropic(
+        model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str
+    ) -> None:
+        return None
+
+    @staticmethod
+    def _translate_adaptive_effort_for_non_adaptive_model(
+        model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str
+    ) -> None:
+        return None
+
+    @staticmethod
+    def _drop_incompatible_temperature_for_thinking(
+        model: str, optional_params: Mapping[str, object], custom_llm_provider: str
+    ) -> None:
+        return None
+
+    def validate_anthropic_messages_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[object],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> tuple[dict[str, str], str | None]:  # mutable-ok: HTTP handler merges forwarded headers
+        resolved_key: Final = get_bigmodel_api_key(api_key)
+        if resolved_key is None:
+            raise AuthenticationError(
+                message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
+            )
+        return {
+            **{key: value for key, value in headers.items() if key.lower() not in ("x-api-key", "authorization")},
+            "x-api-key": resolved_key,
+            "anthropic-version": DEFAULT_ANTHROPIC_API_VERSION,
+            "content-type": "application/json",
+        }, api_base
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        return get_bigmodel_url(api_base, "messages")

litellm/llms/bigmodel/responses/transformation.py

@@ -0,0 +1,263 @@
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
+import httpx
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
+from typing_extensions import Never
+
+from litellm.exceptions import AuthenticationError, UnsupportedParamsError
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.types.llms.openai import (
+    ResponseInputParam,
+    ResponsesAPIOptionalRequestParams,
+    ResponsesAPIResponse,
+    ResponsesAPIStreamingResponse,
+)
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+
+from ..common_utils import bigmodel_business_error, bigmodel_json_body, get_bigmodel_api_key, get_bigmodel_url
+
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_SUPPORTED_PARAMS: Final = (
+    "input",
+    "model",
+    "instructions",
+    "stream",
+    "temperature",
+    "top_p",
+    "max_output_tokens",
+    "tools",
+    "tool_choice",
+    "parallel_tool_calls",
+    "include",
+    "reasoning",
+    "text",
+    "prompt_cache_key",
+    "previous_response_id",
+    "store",
+    "extra_headers",
+    "extra_query",
+    "extra_body",
+    "timeout",
+)
+
+
+_STATUS_BY_BUSINESS_CODE: Final = MappingProxyType(
+    {
+        **dict.fromkeys((1000, 1001, 1003, 1005), 401),
+        **dict.fromkeys((1210, 1211, 1212, 1213, 1214, 1215, 1221, 1222, 1261, 1301), 400),
+        1220: 403,
+        **dict.fromkeys(
+            (1113, 1302, 1305, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321),
+            429,
+        ),
+    }
+)
+
+
+class _RequestedTools(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    tools: JsonValue = None
+    tool_choice: JsonValue = None
+    parallel_tool_calls: JsonValue = None
+
+
+def _requested_tools(logging_obj: Logging) -> _RequestedTools:
+    try:
+        return _RequestedTools.model_validate(logging_obj.model_call_details.get("optional_params") or {})
+    except ValidationError:
+        return _RequestedTools()
+
+
+def _envelope_status_code(code: JsonValue) -> int:
+    if not isinstance(code, int) or isinstance(code, bool):
+        return 500
+    if 400 <= code <= 599:
+        return code
+    return _STATUS_BY_BUSINESS_CODE.get(code, 500)
+
+
+def _normalize_output_item(item: JsonValue) -> JsonValue:
+    if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"):
+        return item
+    content: Final = item.get("content")
+    normalized: Final[JsonValue] = {"summary": [], **item} if item["type"] == "reasoning" else item
+    if not isinstance(content, dict):
+        return normalized
+    return {**normalized, "content": [content]}
+
+
+def _normalize_response(
+    body: Mapping[str, JsonValue],
+    requested: _RequestedTools,
+) -> dict[str, JsonValue]:  # mutable-ok: JSON serialization boundary
+    usage: Final = body.get("usage")
+    output: Final = body.get("output")
+    tools: Final = body.get("tools")
+    tool_choice: Final = body.get("tool_choice")
+    parallel_tool_calls: Final = body.get("parallel_tool_calls")
+    input_tokens: Final = usage.get("input_tokens") if isinstance(usage, dict) else None
+    output_tokens: Final = usage.get("output_tokens") if isinstance(usage, dict) else None
+    normalized_usage: Final = (
+        {
+            "total_tokens": input_tokens + output_tokens,
+            **usage,
+        }
+        if isinstance(usage, dict) and isinstance(input_tokens, int) and isinstance(output_tokens, int)
+        else usage
+    )
+    normalized: Final[JsonValue] = {
+        **body,
+        "parallel_tool_calls": (
+            requested.parallel_tool_calls is not False if parallel_tool_calls is None else parallel_tool_calls
+        ),
+        "tools": (requested.tools or []) if tools is None else tools,
+        "tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice,
+        **({"usage": normalized_usage} if "usage" in body else {}),
+        **({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}),
+    }
+    return normalized
+
+
+def _normalize_stream_part(part: JsonValue) -> JsonValue:
+    if not isinstance(part, dict):
+        return part
+    if part.get("type") == "output_text":
+        return {
+            "annotations": [],
+            "logprobs": None,
+            **part,
+        }
+    if part.get("type") == "reasoning_text":
+        return {"reasoning": part.get("text", ""), **part}
+    return part
+
+
+class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig):
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.BIGMODEL
+
+    def supports_native_file_search(self) -> bool:
+        return False
+
+    def supports_native_websocket(self) -> bool:
+        return False
+
+    def should_fake_stream(
+        self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None
+    ) -> bool:
+        return False
+
+    def may_answer_stream_with_json(self) -> bool:
+        return True
+
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: inherited API requires a list
+        return list(_SUPPORTED_PARAMS)
+
+    def map_openai_params(
+        self,
+        response_api_optional_params: ResponsesAPIOptionalRequestParams,
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: request pipeline adds parameters to the returned body
+        return {key: value for key, value in response_api_optional_params.items() if key in _SUPPORTED_PARAMS}
+
+    def validate_environment(
+        self, headers: Mapping[str, str], model: str, litellm_params: GenericLiteLLMParams | None
+    ) -> dict[str, str]:  # mutable-ok: HTTP handler merges forwarded headers
+        resolved_key: Final = get_bigmodel_api_key(litellm_params.api_key if litellm_params else None)
+        if resolved_key is None:
+            raise AuthenticationError(
+                message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
+            )
+        return {
+            **headers,
+            "Authorization": f"Bearer {resolved_key}",
+            "Content-Type": "application/json",
+        }
+
+    def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str:
+        return get_bigmodel_url(api_base, "responses")
+
+    def transform_response_api_response(
+        self, model: str, raw_response: httpx.Response, logging_obj: Logging
+    ) -> ResponsesAPIResponse:
+        body: Final = bigmodel_json_body(raw_response)
+        if body.get("success") is False:
+            message: Final = str(body.get("msg") or raw_response.text)
+            raise bigmodel_business_error(body.get("code"), message, model, None) or BaseLLMException(
+                message=message,
+                status_code=_envelope_status_code(body.get("code")),
+                headers=raw_response.headers,
+            )
+        normalized: Final = _normalize_response(body, _requested_tools(logging_obj))
+        normalized_response: Final = httpx.Response(
+            status_code=raw_response.status_code,
+            headers={
+                key: value
+                for key, value in raw_response.headers.items()
+                if key.lower() not in ("content-encoding", "content-length")
+            },
+            json=normalized,
+            request=raw_response.request,
+        )
+        return super().transform_response_api_response(model, normalized_response, logging_obj)
+
+    def transform_get_response_api_response(
+        self, raw_response: httpx.Response, logging_obj: Logging
+    ) -> ResponsesAPIResponse:
+        return self.transform_response_api_response("", raw_response, logging_obj)
+
+    def transform_streaming_response(
+        self, model: str, parsed_chunk: Mapping[str, object], logging_obj: Logging
+    ) -> ResponsesAPIStreamingResponse:
+        chunk: Final = _JSON_OBJECT.validate_python(parsed_chunk)
+        response: Final = chunk.get("response")
+        item: Final = chunk.get("item")
+        normalized: Final[JsonValue] = {
+            **chunk,
+            **(
+                {"response": _normalize_response(response, _requested_tools(logging_obj))}
+                if isinstance(response, dict)
+                else {}
+            ),
+            **({"item": _normalize_output_item(item)} if "item" in chunk else {}),
+        }
+        event_model: Final = self.get_event_model_class(str(normalized.get("type")))
+        match chunk.get("type"):
+            case "response.output_text.done":
+                return event_model.model_validate({"logprobs": [], **normalized})
+            case "response.content_part.done":
+                return event_model.model_validate({**normalized, "part": _normalize_stream_part(chunk.get("part"))})
+            case "error":
+                error: Final = chunk.get("error")
+                error_fields: Final = error if isinstance(error, dict) else chunk
+                normalized_error: Final[JsonValue] = {
+                    **normalized,
+                    "error": {"type": "api_error", **error_fields, "code": error_fields.get("code") or "unknown_error"},
+                }
+                return event_model.model_validate(normalized_error)
+            case _:
+                return event_model.model_validate(normalized)
+
+    def transform_cancel_response_api_request(
+        self, response_id: str, api_base: str, litellm_params: GenericLiteLLMParams, headers: Mapping[str, str]
+    ) -> Never:
+        raise UnsupportedParamsError(message="BigModel does not support cancelling responses", status_code=400)
+
+    def transform_compact_response_api_request(
+        self,
+        model: str,
+        input: str | ResponseInputParam,
+        response_api_optional_request_params: Mapping[str, object],
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> Never:
+        raise UnsupportedParamsError(message="BigModel does not support compacting responses", status_code=400)

litellm/llms/cohere/embed/transformation.py

@@ -10,11 +10,13 @@ Convers
 Docs - https://docs.cohere.com/v2/reference/embed
 """
 
-from collections.abc import Iterable, Mapping, Sized
-from typing import Final, Protocol, cast
+from collections.abc import Iterable, Iterator, Mapping, Sequence, Sized
+from enum import Enum
+from itertools import chain, starmap
+from typing import Final, Protocol, TypeAlias, cast
 
 import httpx
-from pydantic import ConfigDict, TypeAdapter
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
 
 import litellm
 from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE
@@ -32,14 +34,117 @@ from litellm.utils import is_base64_encoded
 from ..common_utils import CohereError
 
 _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
-_EMBEDDINGS_BY_TYPE: Final = TypeAdapter(Mapping[str, Iterable[object]], config=ConfigDict(hide_input_in_errors=True))
 _DECODED_JSON: Final = TypeAdapter(object)
 
 
 class _SupportsEncode(Protocol):
     """Tokenizer handle: the embedding usage path only encodes text to measure its token length."""
 
-    def encode(self, text: str, /) -> Sized: ...
+    def encode_ordinary(self, text: str, /) -> Sized: ...
+
+
+class CohereImageTokenBilling(Enum):
+    SEPARATE_FROM_INPUT_TOKENS = "separate_from_input_tokens"
+    FOLDED_INTO_INPUT_TOKENS = "folded_into_input_tokens"
+
+
+class CohereEmbeddingRowShape(Enum):
+    OPENAI = "openai"
+    TYPED = "typed"
+
+
+class _CohereBilledUnits(BaseModel):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+    input_tokens: int | None = None
+    image_tokens: int | None = None
+    images: int | None = None
+
+
+class _CohereEmbedMeta(BaseModel):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+    billed_units: _CohereBilledUnits = Field(default_factory=_CohereBilledUnits)
+
+
+_CohereEmbeddings: TypeAlias = Mapping[str, Sequence[object]] | Sequence[object]
+
+
+class _CohereEmbedBody(BaseModel):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+    embeddings: _CohereEmbeddings
+    meta: _CohereEmbedMeta = Field(default_factory=_CohereEmbedMeta)
+
+
+def _embedding_rows(vectors: Sequence[object]) -> Iterator[Mapping[str, object]]:
+    return ({"object": "embedding", "index": index, "embedding": vector} for index, vector in enumerate(vectors))
+
+
+def _typed_embedding_rows(embedding_type: str, vectors: Sequence[object]) -> Iterator[Mapping[str, object]]:
+    return ({**row, "type": embedding_type} for row in _embedding_rows(vectors))
+
+
+def _response_rows(embeddings: _CohereEmbeddings, row_shape: CohereEmbeddingRowShape) -> Iterable[Mapping[str, object]]:
+    if not isinstance(embeddings, Mapping):
+        return _embedding_rows(embeddings)
+    match row_shape:
+        case CohereEmbeddingRowShape.OPENAI:
+            return chain.from_iterable(map(_embedding_rows, embeddings.values()))
+        case CohereEmbeddingRowShape.TYPED:
+            return chain.from_iterable(starmap(_typed_embedding_rows, embeddings.items()))
+
+
+def _prompt_usage(prompt_tokens: int, prompt_tokens_details: PromptTokensDetailsWrapper | None) -> Usage:
+    return Usage(
+        prompt_tokens=prompt_tokens,
+        completion_tokens=0,
+        total_tokens=prompt_tokens,
+        prompt_tokens_details=prompt_tokens_details,
+    )
+
+
+def _billed_usage(billed_units: _CohereBilledUnits, image_billing: CohereImageTokenBilling) -> Usage:
+    match image_billing:
+        case CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS:
+            image_units: Final = (billed_units.image_tokens or 0) + (billed_units.images or 0)
+            return _prompt_usage(
+                (billed_units.input_tokens or 0) + image_units,
+                PromptTokensDetailsWrapper(
+                    text_tokens=billed_units.input_tokens,
+                    image_tokens=image_units or None,
+                    image_count=billed_units.images,
+                ),
+            )
+        case CohereImageTokenBilling.FOLDED_INTO_INPUT_TOKENS:
+            return _prompt_usage(billed_units.input_tokens or 0, None)
+
+
+def _cohere_embed_usage(
+    billed_units: _CohereBilledUnits,
+    input_texts: Iterable[str],
+    encoding: _SupportsEncode,
+    image_billing: CohereImageTokenBilling,
+) -> Usage:
+    if billed_units.input_tokens is None and billed_units.image_tokens is None and billed_units.images is None:
+        return _prompt_usage(sum(len(encoding.encode_ordinary(text)) for text in input_texts), None)
+    return _billed_usage(billed_units, image_billing)
+
+
+def cohere_embedding_response(
+    body: Mapping[str, object],
+    model: str | None,
+    encoding: _SupportsEncode,
+    input_texts: Iterable[str],
+    image_billing: CohereImageTokenBilling,
+    row_shape: CohereEmbeddingRowShape,
+) -> EmbeddingResponse:
+    cohere_body: Final = _CohereEmbedBody.model_validate(body)
+    return EmbeddingResponse(
+        model=model,
+        data=list(_response_rows(cohere_body.embeddings, row_shape)),
+        usage=_cohere_embed_usage(cohere_body.meta.billed_units, input_texts, encoding, image_billing),
+    )
 
 
 class CohereEmbeddingConfig(BaseEmbeddingConfig):
@@ -145,44 +250,16 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig):
             ),
         )
 
-    def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage:
-        input_tokens = 0
-
-        text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens")
-
-        image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images")
-
-        prompt_tokens_details: PromptTokensDetailsWrapper | None = None
-        if image_tokens is None and text_tokens is None:
-            for text in input:
-                input_tokens += len(encoding.encode(text))
-        else:
-            prompt_tokens_details = PromptTokensDetailsWrapper(
-                image_tokens=image_tokens,
-                text_tokens=text_tokens,
-            )
-            if image_tokens:
-                input_tokens += image_tokens
-            if text_tokens:
-                input_tokens += text_tokens
-
-        return Usage(
-            prompt_tokens=input_tokens,
-            completion_tokens=0,
-            total_tokens=input_tokens,
-            prompt_tokens_details=prompt_tokens_details,
-        )
-
-    def _transform_response(
+    def transform_native_embed_response(
         self,
         response: httpx.Response,
         api_key: str | None,
         logging_obj: LiteLLMLoggingObj,
-        data: dict | CohereEmbeddingRequest,
-        model_response: EmbeddingResponse,
+        data: Mapping[str, object] | CohereEmbeddingRequest,
         model: str,
         encoding: _SupportsEncode,
-        input: list,
+        input: Sequence[str],
+        row_shape: CohereEmbeddingRowShape,
     ) -> EmbeddingResponse:
         response_json: Final = _DECODED_JSON.validate_python(response.json())
         ## LOGGING
@@ -192,38 +269,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig):
             additional_args={"complete_input_dict": data},
             original_response=response_json,
         )
-        """
-            response 
-            {
-                'object': "list",
-                'data': [
-                
-                ]
-                'model', 
-                'usage'
-            }
-        """
-        payload: Final = _JSON_OBJECT.validate_python(response_json)
-        embeddings: Final = _EMBEDDINGS_BY_TYPE.validate_python(payload["embeddings"])
-        output_data: Final = []
-        for k, embedding_list in embeddings.items():
-            for idx, embedding in enumerate(embedding_list):
-                output_data.append({"object": "embedding", "index": idx, "embedding": embedding})
-        model_response.object = "list"
-        model_response.data = output_data
-        model_response.model = model
-        input_tokens = 0
-        for text in input:
-            input_tokens += len(encoding.encode(text))
-
-        setattr(
-            model_response,
-            "usage",
-            self._calculate_usage(input, encoding, dict(_JSON_OBJECT.validate_python(payload.get("meta", {})))),
+        return cohere_embedding_response(
+            _JSON_OBJECT.validate_python(response_json),
+            model,
+            encoding,
+            input,
+            CohereImageTokenBilling.SEPARATE_FROM_INPUT_TOKENS,
+            row_shape,
         )
 
-        return model_response
-
     def transform_embedding_response(
         self,
         model: str,
@@ -235,15 +289,15 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig):
         optional_params: dict,
         litellm_params: dict,
     ) -> EmbeddingResponse:
-        return self._transform_response(
+        return self.transform_native_embed_response(
             response=raw_response,
             api_key=api_key,
             logging_obj=logging_obj,
             data=request_data,
-            model_response=model_response,
             model=model,
             encoding=litellm.encoding,
             input=logging_obj.model_call_details["input"],
+            row_shape=CohereEmbeddingRowShape.OPENAI,
         )
 
     def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:

litellm/llms/custom_httpx/http_handler.py

@@ -764,18 +764,23 @@ class AsyncHTTPHandler:
         follow_redirects: bool | None = None,
         timeout: float | httpx.Timeout | None = None,
         max_response_bytes: int | None = None,
+        stream: bool = False,
     ):
         # Set follow_redirects to UseClientDefault if None
         _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
+        query: Final = httpx.QueryParams(params).merge(HTTPHandler.extract_query_params(url))
+        request_headers: Final = httpx.Headers(headers)
 
-        params = params or {}
-        params.update(HTTPHandler.extract_query_params(url))
+        if stream:
+            return await self._send_streaming_get(
+                url, params=query, headers=request_headers, follow_redirects=follow_redirects, timeout=timeout
+            )
 
         if max_response_bytes is not None:
             return await self._get_with_response_limit(
                 url,
-                params=httpx.QueryParams(params),
-                headers=httpx.Headers(headers),
+                params=query,
+                headers=request_headers,
                 max_bytes=max_response_bytes,
                 follow_redirects=self.client.follow_redirects if follow_redirects is None else follow_redirects,
                 timeout=self.client.timeout if timeout is None else httpx.Timeout(timeout),
@@ -783,13 +788,41 @@ class AsyncHTTPHandler:
 
         response: Final = await self.client.get(
             url,
-            params=params,
-            headers=headers,
+            params=query,
+            headers=request_headers,
             follow_redirects=_follow_redirects,
             timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT,
         )
         return response
 
+    async def _send_streaming_get(
+        self,
+        url: str,
+        *,
+        params: httpx.QueryParams,
+        headers: httpx.Headers,
+        follow_redirects: bool | None,
+        timeout: float | httpx.Timeout | None,
+    ) -> httpx.Response:
+        start_time: Final = time.time()
+        request: Final = self.client.build_request(
+            "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout
+        )
+        try:
+            response: Final = await self.client.send(
+                request,
+                stream=True,
+                follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects,
+            )
+            _anchor_handler_to(response, self)
+            response.raise_for_status()
+        except httpx.TimeoutException as e:
+            _raise_timeout(e, start_time)
+        except httpx.HTTPStatusError as e:
+            await _raise_masked_async_error(e, True)
+        else:
+            return response
+
     async def _get_with_response_limit(
         self,
         url: str,
@@ -1468,15 +1501,20 @@ class HTTPHandler:
         headers: Mapping[str, Any] | None = None,
         follow_redirects: bool | None = None,
         timeout: float | httpx.Timeout | None = None,
+        stream: bool = False,
     ):
         # Set follow_redirects to UseClientDefault if None
         _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
-        params = params or {}
-        params.update(self.extract_query_params(url))
+        query: Final = httpx.QueryParams(params).merge(self.extract_query_params(url))
+
+        if stream:
+            return self._send_streaming_get(
+                url, params=query, headers=headers, follow_redirects=follow_redirects, timeout=timeout
+            )
 
         response: Final = self.client.get(
             url,
-            params=params,
+            params=query,
             headers=headers,
             follow_redirects=_follow_redirects,
             timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT,
@@ -1484,6 +1522,34 @@ class HTTPHandler:
 
         return response
 
+    def _send_streaming_get(
+        self,
+        url: str,
+        *,
+        params: httpx.QueryParams,
+        headers: Mapping[str, str] | None,
+        follow_redirects: bool | None,
+        timeout: float | httpx.Timeout | None,
+    ) -> httpx.Response:
+        start_time: Final = time.time()
+        request: Final = self.client.build_request(
+            "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout
+        )
+        try:
+            response: Final = self.client.send(
+                request,
+                stream=True,
+                follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects,
+            )
+            _anchor_handler_to(response, self)
+            response.raise_for_status()
+        except httpx.TimeoutException as e:
+            _raise_timeout(e, start_time)
+        except httpx.HTTPStatusError as e:
+            _raise_masked_sync_error(e, True)
+        else:
+            return response
+
     @staticmethod
     def extract_query_params(url: str) -> dict[str, str]:
         """

litellm/llms/custom_httpx/llm_http_handler.py

@@ -50,7 +50,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
     SUBTITLE_RESPONSE_FORMATS,
     synthesize_subtitle_document,
 )
-from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params
+from litellm.litellm_core_utils.core_helpers import (
+    as_str_mapping,
+    qualify_agentic_followup_model,
+    set_provider_response_headers_in_hidden_params,
+)
 from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
 from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
 from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
@@ -367,12 +371,37 @@ class _SignedBodyKwargs(TypedDict, total=False):
     json: ReadOnly[dict[str, object]]
 
 
+def _is_sized_json_body(response: httpx.Response) -> bool:
+    headers: Final = httpx.Headers(response.headers)
+    media_types: Final = (value.split(";")[0].strip().lower() for value in headers.get_list("content-type"))
+    return "content-length" in headers and "application/json" in media_types
+
+
+def _is_json_document(content: bytes) -> bool:
+    try:
+        return isinstance(json.loads(content), dict)
+    except ValueError:
+        return False
+
+
 def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -> _SignedBodyKwargs:
     if signed_body is not None:
         return {"data": signed_body}
     return {"json": data}
 
 
+def _answered_json(response: httpx.Response) -> bool:
+    return "application/json" in response.headers.get("content-type", "").lower()
+
+
+def _stream_query_params(stream: bool, starting_after: int | None) -> Mapping[str, object]:
+    if not stream:
+        return {}
+    if starting_after is None:
+        return {"stream": "true"}
+    return {"stream": "true", "starting_after": starting_after}
+
+
 def _google_genai_streaming_hidden_params(
     *,
     api_base: str,
@@ -2552,6 +2581,7 @@ class BaseLLMHTTPHandler:
                 "preset_cache_key": None,
                 "stream_response": {},
                 "model_info": kwargs.get("model_info"),
+                **litellm_params.per_second_pricing(),
                 **vertex_location_params,
                 **anthropic_messages_optional_request_params,
             },
@@ -3025,6 +3055,17 @@ class BaseLLMHTTPHandler:
             },
         )
 
+        replay_as_stream: Final = partial(
+            MockResponsesAPIStreamingIterator,
+            model=model,
+            logging_obj=logging_obj,
+            responses_api_provider_config=responses_api_provider_config,
+            litellm_metadata=litellm_metadata,
+            custom_llm_provider=custom_llm_provider,
+            request_data=request_context,
+            call_type=CallTypes.responses.value,
+        )
+
         try:
             if is_stream_request:
                 response = sync_httpx_client.post(
@@ -3036,7 +3077,13 @@ class BaseLLMHTTPHandler:
                 )
                 if caller_requested_stream:
                     if fake_stream is True:
-                        return MockResponsesAPIStreamingIterator(
+                        return replay_as_stream(response=response)
+                    if not (
+                        responses_api_provider_config.may_answer_stream_with_json()
+                        and _is_sized_json_body(response)
+                        and _is_json_document(response.read())
+                    ):
+                        return SyncResponsesAPIStreamingIterator(
                             response=response,
                             model=model,
                             logging_obj=logging_obj,
@@ -3046,17 +3093,6 @@ class BaseLLMHTTPHandler:
                             request_data=request_context,
                             call_type=CallTypes.responses.value,
                         )
-
-                    return SyncResponsesAPIStreamingIterator(
-                        response=response,
-                        model=model,
-                        logging_obj=logging_obj,
-                        responses_api_provider_config=responses_api_provider_config,
-                        litellm_metadata=litellm_metadata,
-                        custom_llm_provider=custom_llm_provider,
-                        request_data=request_context,
-                        call_type=CallTypes.responses.value,
-                    )
                 response.read()
             else:
                 response = sync_httpx_client.post(
@@ -3071,6 +3107,9 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if is_stream_request and caller_requested_stream:
+            return replay_as_stream(response=response)
+
         initial_response: Final = responses_api_provider_config.transform_response_api_response(
             model=model,
             raw_response=response,
@@ -3226,6 +3265,17 @@ class BaseLLMHTTPHandler:
             },
         )
 
+        replay_as_stream: Final = partial(
+            MockResponsesAPIStreamingIterator,
+            model=model,
+            logging_obj=logging_obj,
+            responses_api_provider_config=responses_api_provider_config,
+            litellm_metadata=litellm_metadata,
+            custom_llm_provider=custom_llm_provider,
+            request_data=request_context,
+            call_type=CallTypes.responses.value,
+        )
+
         try:
             if is_stream_request:
                 response = await async_httpx_client.post(
@@ -3239,7 +3289,13 @@ class BaseLLMHTTPHandler:
 
                 if caller_requested_stream:
                     if fake_stream is True:
-                        return MockResponsesAPIStreamingIterator(
+                        return replay_as_stream(response=response)
+                    if not (
+                        responses_api_provider_config.may_answer_stream_with_json()
+                        and _is_sized_json_body(response)
+                        and _is_json_document(await response.aread())
+                    ):
+                        return ResponsesAPIStreamingIterator(
                             response=response,
                             model=model,
                             logging_obj=logging_obj,
@@ -3249,17 +3305,6 @@ class BaseLLMHTTPHandler:
                             request_data=request_context,
                             call_type=CallTypes.responses.value,
                         )
-
-                    return ResponsesAPIStreamingIterator(
-                        response=response,
-                        model=model,
-                        logging_obj=logging_obj,
-                        responses_api_provider_config=responses_api_provider_config,
-                        litellm_metadata=litellm_metadata,
-                        custom_llm_provider=custom_llm_provider,
-                        request_data=request_context,
-                        call_type=CallTypes.responses.value,
-                    )
                 await response.aread()
             else:
                 response = await async_httpx_client.post(
@@ -3276,6 +3321,9 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if is_stream_request and caller_requested_stream:
+            return replay_as_stream(response=response)
+
         initial_response: Final = responses_api_provider_config.transform_response_api_response(
             model=model,
             raw_response=response,
@@ -3493,12 +3541,20 @@ class BaseLLMHTTPHandler:
         logging_obj: LiteLLMLoggingObj,
         custom_llm_provider: str | None = None,
         extra_headers: dict[str, object] | None = None,
+        extra_query: Mapping[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
+        stream: bool = False,
+        starting_after: int | None = None,
+        litellm_metadata: Mapping[str, object] = MappingProxyType({}),
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         _is_async: bool = False,
         shared_session: Optional["ClientSession"] = None,
-    ) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]:
+    ) -> (
+        ResponsesAPIResponse
+        | SyncResponsesAPIStreamingIterator
+        | Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator]
+    ):
         """
         Get a response by ID
         Uses GET /v1/responses/{response_id} endpoint in the responses API
@@ -3511,7 +3567,11 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
                 custom_llm_provider=custom_llm_provider,
                 extra_headers=extra_headers,
+                extra_query=extra_query,
                 extra_body=extra_body,
+                stream=stream,
+                starting_after=starting_after,
+                litellm_metadata=litellm_metadata,
                 timeout=timeout,
                 client=client,
                 shared_session=shared_session,
@@ -3540,20 +3600,23 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             headers=headers,
         )
+        query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)}
 
         ## LOGGING
         logging_obj.pre_call(
             input="",
             api_key="",
             additional_args={
-                "complete_input_dict": data,
+                "complete_input_dict": query,
                 "api_base": api_base,
                 "headers": headers,
             },
         )
 
         try:
-            response: Final = sync_httpx_client.get(url=url, headers=headers, params=data)
+            response: Final = sync_httpx_client.get(
+                url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None
+            )
             response.raise_for_status()
         except Exception as e:
             raise self._handle_error(
@@ -3561,6 +3624,20 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if stream and not _answered_json(response):
+            return SyncResponsesAPIStreamingIterator(
+                response=response,
+                model="",
+                logging_obj=logging_obj,
+                responses_api_provider_config=responses_api_provider_config,
+                litellm_metadata=dict(litellm_metadata),
+                custom_llm_provider=custom_llm_provider,
+                request_data={"litellm_params": dict(litellm_params)},
+                replays_stored_stream=True,
+            )
+
+        if stream:
+            response.read()
         return responses_api_provider_config.transform_get_response_api_response(
             raw_response=response,
             logging_obj=logging_obj,
@@ -3574,11 +3651,15 @@ class BaseLLMHTTPHandler:
         logging_obj: LiteLLMLoggingObj,
         custom_llm_provider: str | None = None,
         extra_headers: dict[str, object] | None = None,
+        extra_query: Mapping[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
+        stream: bool = False,
+        starting_after: int | None = None,
+        litellm_metadata: Mapping[str, object] = MappingProxyType({}),
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         shared_session: Optional["ClientSession"] = None,
-    ) -> ResponsesAPIResponse:
+    ) -> ResponsesAPIResponse | ResponsesAPIStreamingIterator:
         """
         Async version of get_responses
         """
@@ -3616,20 +3697,23 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             headers=headers,
         )
+        query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)}
 
         ## LOGGING
         logging_obj.pre_call(
             input="",
             api_key="",
             additional_args={
-                "complete_input_dict": data,
+                "complete_input_dict": query,
                 "api_base": api_base,
                 "headers": headers,
             },
         )
 
         try:
-            response: Final = await async_httpx_client.get(url=url, headers=headers, params=data)
+            response: Final = await async_httpx_client.get(
+                url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None
+            )
             response.raise_for_status()
         except Exception as e:
             verbose_logger.debug("Error retrieving response: %s", e)
@@ -3638,6 +3722,20 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if stream and not _answered_json(response):
+            return ResponsesAPIStreamingIterator(
+                response=response,
+                model="",
+                logging_obj=logging_obj,
+                responses_api_provider_config=responses_api_provider_config,
+                litellm_metadata=dict(litellm_metadata),
+                custom_llm_provider=custom_llm_provider,
+                request_data={"litellm_params": dict(litellm_params)},
+                replays_stored_stream=True,
+            )
+
+        if stream:
+            await response.aread()
         return responses_api_provider_config.transform_get_response_api_response(
             raw_response=response,
             logging_obj=logging_obj,
@@ -6182,9 +6280,7 @@ class BaseLLMHTTPHandler:
         if patch.messages is None:
             raise ValueError("Agentic loop plan missing patched messages")
 
-        full_model_name = patch.model or model
-        if "/" not in full_model_name:
-            full_model_name = f"{custom_llm_provider}/{full_model_name}"
+        full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider)
 
         optional_params_for_followup: Final = dict(optional_params)
         optional_params_for_followup.update(patch.optional_params)

litellm/llms/databricks/chat/transformation.py

@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
 import httpx
 from pydantic import BaseModel, ConfigDict, TypeAdapter
 
+from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_thinking
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
 from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
     handle_invalid_parallel_tool_calls,
@@ -514,6 +515,14 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         self.update_optional_params_with_thinking_tokens(
             non_default_params=non_default_params, optional_params=mapped_params
         )
+        gated_thinking: Final = strip_beta_gated_thinking(
+            mapped_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # mapped_params is the base config's bare dict
+            frozenset(),
+            drop_params=drop_params,
+        )
+        if gated_thinking.stripped:
+            log_beta_gated_strips(gated_thinking.stripped, model=model, provider="databricks")
+            mapped_params["thinking"] = gated_thinking.thinking
 
         return mapped_params
 

litellm/llms/databricks/messages/transformation.py

@@ -144,7 +144,7 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig)
             ("content-type" if key.lower() == "content-type" else key): value
             for key, value in {**auth_headers, **client_anthropic_headers}.items()
         }
-        finalized_headers: Final[dict[str, str]] = self._finalize_messages_headers(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the shared helper returns a bare dict  # mutable-ok: becomes the outbound headers
+        finalized_headers: Final[dict[str, str]] = self.finalize_messages_headers(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the shared helper returns a bare dict  # mutable-ok: becomes the outbound headers
             headers=normalized_headers,
             optional_params=optional_params,
             messages=messages,
@@ -191,6 +191,8 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig)
             litellm_params=litellm_params,
             headers=headers,
         )
-        self._normalize_system_role_messages(anthropic_messages_request, model=model)  # pyright: ignore[reportUnknownMemberType]  # the shared helper takes a bare dict
+        self._normalize_system_role_messages(  # pyright: ignore[reportUnknownMemberType]  # the shared helper takes a bare dict
+            anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None)
+        )
         self._remove_scope_from_cache_control(anthropic_messages_request)  # pyright: ignore[reportUnknownMemberType]  # the shared helper takes a bare dict
         return anthropic_messages_request

litellm/llms/fireworks_ai/responses/transformation.py

@@ -1,10 +1,9 @@
-from collections.abc import Mapping, Sequence
+from collections.abc import Mapping
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final
 from urllib.parse import unquote
 
 import httpx
-from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam
 
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.fireworks_ai.common_utils import (
@@ -15,12 +14,18 @@ from litellm.llms.fireworks_ai.common_utils import (
     with_fireworks_session_affinity,
     without_caller_user,
 )
+from litellm.llms.openai.responses.instruction_items import developer_items_as_system, instruction_items_folded
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import ResponseInputParam
 from litellm.types.responses.main import DeleteResponseResult
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
+from litellm.utils import (
+    declared_mid_conversation_system_support,
+    deployment_declared_mid_conversation_system_support,
+    deployment_model_info,
+)
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -45,90 +50,6 @@ def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str,
     )
 
 
-_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"})
-
-
-def _role(item: ResponseInputItemParam) -> str | None:
-    match item:
-        case {"role": str(role)}:
-            return role
-        case _:
-            return None
-
-
-def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam:
-    if "role" not in item or item["role"] != "developer":
-        return item
-    return EasyInputMessageParam(role="system", content=item["content"], type="message")
-
-
-def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam:
-    if isinstance(input, str):
-        return input
-    return [_developer_item_as_system(item) for item in input]
-
-
-def _text_part(part: ResponseInputContentParam) -> str | None:
-    match part:
-        case {"type": "input_text", "text": str(text)}:
-            return text
-        case _:
-            return None
-
-
-def _text_only_content(item: ResponseInputItemParam) -> str | None:
-    match item:
-        case {"role": "system" | "developer", "content": str(text)}:
-            return text
-        case {"role": "system" | "developer", "content": [*parts]}:
-            texts: Final = tuple(map(_text_part, parts))
-            return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text)
-        case _:
-            return None
-
-
-def _leading_instruction_block_length(roles: Sequence[str | None]) -> int:
-    return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles))
-
-
-def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int:
-    last_conversation_index: Final = next(
-        (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES),
-        None,
-    )
-    if last_conversation_index is None or roles[last_conversation_index] != "assistant":
-        return len(roles)
-    return last_conversation_index + 1
-
-
-def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]:
-    leading_length: Final = _leading_instruction_block_length(roles)
-    closing_start: Final = _closing_instruction_block_start(roles, leading_length)
-    return tuple(
-        index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer"
-    )
-
-
-def _with_instruction_items_folded(
-    input: str | ResponseInputParam, instructions: str | None
-) -> tuple[str | None, str | ResponseInputParam]:
-    if isinstance(input, str):
-        return instructions, input
-    items: Final = tuple(input)
-    folded: Final = MappingProxyType(
-        {
-            index: text
-            for index in _hoisted_indices(tuple(map(_role, items)))
-            if (text := _text_only_content(items[index])) is not None
-        }
-    )
-    joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk)
-    return (
-        instructions if not folded else joined or None,
-        [_developer_item_as_system(item) for index, item in enumerate(items) if index not in folded],
-    )
-
-
 class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     @property
     def custom_llm_provider(self) -> LlmProviders:
@@ -158,29 +79,28 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         self,
         model: str,
         input: str | ResponseInputParam,
-        response_api_optional_request_params: dict,  # mutable-ok: overrides the base class signature
+        response_api_optional_request_params: dict[str, object],  # mutable-ok: overrides the base class signature
         litellm_params: GenericLiteLLMParams,
-        headers: dict,  # mutable-ok: overrides the base class signature
-    ) -> dict:  # mutable-ok: overrides the base class signature
-        instructions_param: Final[object] = response_api_optional_request_params.get("instructions")
+        headers: dict[str, str],  # mutable-ok: overrides the base class signature
+    ) -> dict[str, object]:  # mutable-ok: overrides the base class signature
         validated_input: Final = self._validate_input_param(input)
-        instructions, folded_input = (
-            _with_instruction_items_folded(validated_input, instructions_param)
-            if isinstance(instructions_param, str | None)
-            else (instructions_param, _developer_items_as_system(validated_input))
-        )
-        instruction_entries: Final = () if instructions is None else (("instructions", instructions),)
-        folded_params: Final = {
-            key: value
-            for key, value in (
-                *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"),
-                *instruction_entries,
+        model_info: Final = deployment_model_info(litellm_params)
+        declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info)
+        folded: Final = (
+            None
+            if deployment_declared_mid_conversation_system_support(model_info) is True
+            else instruction_items_folded(
+                validated_input, response_api_optional_request_params, "user" if declared is False else "system"
             )
-        }
+        )
+        placed_input, placed_params = folded or (
+            developer_items_as_system(validated_input),
+            response_api_optional_request_params,
+        )
         request: Final = super().transform_responses_api_request(
             model=resolve_fireworks_resource_name(model),
-            input=folded_input,
-            response_api_optional_request_params=folded_params,
+            input=placed_input,
+            response_api_optional_request_params=dict(placed_params),
             litellm_params=litellm_params,
             headers=headers,
         )

litellm/llms/github_copilot/authenticator.py

@@ -1,11 +1,12 @@
 import json
 import os
 import time
+from dataclasses import dataclass
 from datetime import datetime
-from typing import Any, Final
+from typing import Any, Final, TypeVar
 
 import httpx
-from pydantic import ConfigDict, TypeAdapter, with_config
+from pydantic import BaseModel, ConfigDict, TypeAdapter, with_config
 from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm._logging import verbose_logger
@@ -13,7 +14,6 @@ from litellm.litellm_core_utils.asyncify import can_block_current_thread
 from litellm.llms.custom_httpx.http_handler import get_httpx_client
 
 from .common_utils import (
-    APIKeyExpiredError,
     GetAccessTokenError,
     GetAPIKeyError,
     GetDeviceCodeError,
@@ -27,6 +27,44 @@ DEFAULT_GITHUB_ACCESS_TOKEN_URL: Final = "https://github.com/login/oauth/access_
 DEFAULT_GITHUB_API_KEY_URL: Final = "https://api.github.com/copilot_internal/v2/token"
 
 
+class CopilotEndpoints(BaseModel):
+    model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True)
+
+    api: str | None = None
+
+
+class CopilotCachedKey(BaseModel):
+    model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True)
+
+    token: str
+    expires_at: float
+    endpoints: CopilotEndpoints | None = None
+
+
+class CopilotCachedEndpoints(BaseModel):
+    model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True)
+
+    endpoints: CopilotEndpoints | None = None
+
+
+ApiKeyFileView = TypeVar("ApiKeyFileView", CopilotCachedKey, CopilotCachedEndpoints)
+
+
+@dataclass(frozen=True, slots=True)
+class _UnsavedApiKey:
+    access_token: str
+    key: CopilotCachedKey
+
+
+_UNSAVED_API_KEYS: Final[dict[str, _UnsavedApiKey]] = {}  # mutable-ok: refreshed keys api-key.json could not hold
+
+
+def _read_token_file(path: str) -> str:
+    with open(path, "rb") as f:
+        raw: Final = f.read()
+    return raw.decode(json.detect_encoding(raw))
+
+
 @with_config(ConfigDict(extra="allow", strict=True))
 class _DeviceCode(TypedDict):
     device_code: ReadOnly[str]
@@ -87,12 +125,13 @@ class Authenticator:
             GetAccessTokenError: If unable to obtain an access token after retries.
         """
         try:
-            with open(self.access_token_file, "r") as f:
-                access_token = f.read().strip()
-                if access_token:
-                    return access_token
+            saved_token: Final = _read_token_file(self.access_token_file).strip()
+            if saved_token:
+                return saved_token
         except OSError:
             verbose_logger.warning("No existing access token found or error reading file")
+        except ValueError as e:
+            verbose_logger.warning("Ignoring %s, not a text access token: %s", self.access_token_file, e)
 
         if not can_block_current_thread():
             raise GetAccessTokenError(
@@ -135,64 +174,83 @@ class Authenticator:
         Raises:
             GetAPIKeyError: If unable to obtain an API key.
         """
-        try:
-            with open(self.api_key_file, "r") as f:
-                api_key_info = json.load(f)
-                if api_key_info.get("expires_at", 0) > datetime.now().timestamp():
-                    return api_key_info.get("token")
-                else:
-                    verbose_logger.warning("API key expired, refreshing")
-                    raise APIKeyExpiredError(
-                        message="API key expired",
-                        status_code=401,
-                    )
-        except OSError:
-            verbose_logger.warning("No API key file found or error opening file")
-        except (json.JSONDecodeError, KeyError) as e:
-            verbose_logger.warning("Error reading API key from file: %s", e)
-        except APIKeyExpiredError:
-            pass  # Already logged in the try block
+        unsaved: Final = self._unsaved_api_key()
+        if unsaved is not None:
+            return unsaved.token
+        cached: Final = self._read_api_key_file(CopilotCachedKey)
+        if cached is not None and cached.expires_at > datetime.now().timestamp():
+            return cached.token
+        if cached is not None:
+            verbose_logger.warning("API key expired, refreshing")
 
         try:
-            api_key_info = self._refresh_api_key()
-            with open(self.api_key_file, "w") as f:
-                json.dump(api_key_info, f)
-            token: Final = api_key_info.get("token")
-            if token:
-                return token
-            else:
-                raise GetAPIKeyError(
-                    message="API key response missing token",
-                    status_code=401,
-                )
-        except OSError as e:
-            verbose_logger.error("Error saving API key to file: %s", e)
-            raise GetAPIKeyError(
-                message=f"Failed to save API key: {e}",
-                status_code=500,
-            )
+            api_key_info: Final = self._refresh_api_key()
         except RefreshAPIKeyError as e:
             raise GetAPIKeyError(
                 message=f"Failed to refresh API key: {e}",
                 status_code=401,
+            ) from e
+        serialized: Final = json.dumps(api_key_info)
+        try:
+            with open(self.api_key_file, "w") as f:
+                f.write(serialized)
+        except OSError as e:
+            verbose_logger.warning(
+                "Keeping the refreshed API key in memory, it could not be saved to %s: %s", self.api_key_file, e
             )
+            self._keep_unsaved_api_key(serialized)
+        token: Final = api_key_info.get("token")
+        if not token:
+            raise GetAPIKeyError(
+                message="API key response missing token",
+                status_code=401,
+            )
+        return token
 
     def get_api_base(self) -> str | None:
         """
-        Get the API endpoint from the api-key.json file.
+        Get the API endpoint of the current API key.
 
         Returns:
             Optional[str]: The GitHub Copilot API endpoint, or None if not found.
         """
-        try:
-            with open(self.api_key_file, "r") as f:
-                api_key_info: Final = json.load(f)
-                endpoints: Final = api_key_info.get("endpoints", {})
-                api_endpoint: Final = endpoints.get("api")
-                return api_endpoint
-        except (OSError, json.JSONDecodeError, KeyError) as e:
-            verbose_logger.warning("Error reading API endpoint from file: %s", e)
+        cached: Final = self._unsaved_api_key() or self._read_api_key_file(CopilotCachedEndpoints)
+        if cached is None or cached.endpoints is None:
             return None
+        return cached.endpoints.api
+
+    def _read_api_key_file(self, view: type[ApiKeyFileView]) -> ApiKeyFileView | None:
+        try:
+            return view.model_validate_json(_read_token_file(self.api_key_file))
+        except OSError:
+            verbose_logger.warning("No API key file found or error opening file")
+            return None
+        except ValueError as e:
+            verbose_logger.warning("Ignoring %s, not a Copilot API key file: %s", self.api_key_file, e)
+            return None
+
+    def _stored_access_token(self) -> str | None:
+        try:
+            access_token: Final = _read_token_file(self.access_token_file).strip()
+        except (OSError, ValueError):
+            return None
+        return access_token or None
+
+    def _unsaved_api_key(self) -> CopilotCachedKey | None:
+        unsaved: Final = _UNSAVED_API_KEYS.get(self.api_key_file)
+        if unsaved is None or unsaved.key.expires_at <= time.time():
+            return None
+        return unsaved.key if unsaved.access_token == self._stored_access_token() else None
+
+    def _keep_unsaved_api_key(self, serialized: str) -> None:
+        access_token: Final = self._stored_access_token()
+        if access_token is None:
+            return
+        try:
+            key: Final = CopilotCachedKey.model_validate_json(serialized)
+        except ValueError:
+            return
+        _UNSAVED_API_KEYS[self.api_key_file] = _UnsavedApiKey(access_token=access_token, key=key)
 
     def _refresh_api_key(self) -> dict[str, Any]:
         """

litellm/llms/hosted_vllm/responses/transformation.py

@@ -9,10 +9,13 @@ the chat completions → responses conversion pipeline.
 from typing import Final
 
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.openai.responses.instruction_items import instruction_items_folded
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import ResponseInputParam
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
+from litellm.utils import declared_mid_conversation_system_support, deployment_model_info
 
 
 class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig):
@@ -69,6 +72,31 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
         return join_api_base(api_base, "v1/responses")
 
+    def transform_responses_api_request(
+        self,
+        model: str,
+        input: str | ResponseInputParam,
+        response_api_optional_request_params: dict[str, object],  # mutable-ok: overrides the base class signature
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: overrides the base class signature
+    ) -> dict[str, object]:  # mutable-ok: overrides the base class signature
+        declared: Final = declared_mid_conversation_system_support(
+            model, self.custom_llm_provider.value, deployment_model_info(litellm_params)
+        )
+        folded: Final = (
+            instruction_items_folded(self._validate_input_param(input), response_api_optional_request_params, "user")
+            if declared is False
+            else None
+        )
+        placed_input, placed_params = folded or (input, response_api_optional_request_params)
+        return super().transform_responses_api_request(
+            model=model,
+            input=placed_input,
+            response_api_optional_request_params=dict(placed_params),
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
     def supports_native_websocket(self) -> bool:
         """Hosted vLLM does not support native WebSocket for Responses API"""
         return False

litellm/llms/manus/responses/transformation.py

@@ -241,7 +241,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         """
         Transform the get response API request into a URL and data.
 
@@ -252,7 +252,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
         """
         encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
         url: Final = f"{api_base}/{encoded_response_id}"
-        data: Final[dict] = {}
+        data: Final[dict[str, str]] = {}
         return url, data
 
     def transform_get_response_api_response(

litellm/llms/openai/chat/gpt_reasoning_transformation.py

@@ -5,8 +5,10 @@ from collections.abc import Collection
 from typing import ClassVar, Final
 
 import litellm
+from litellm._logging import verbose_logger
 from litellm.utils import (
     declared_value_factory,
+    has_catalog_entry,
     is_explicitly_disabled_factory,
     supports_factory,
 )
@@ -16,6 +18,25 @@ from .gpt_transformation import OpenAIGPTConfig
 _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)")
 
 
+def refused_reasoning_effort_warning(*, suggest_declared_levels: bool, suggest_base_model: bool) -> str:
+    return "".join(
+        (
+            "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level "
+            "supported for %r.",
+            (
+                " If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels."
+                if suggest_declared_levels
+                else ""
+            ),
+            (
+                " If the deployment name is not its underlying model, set model_info.base_model to that model."
+                if suggest_base_model
+                else ""
+            ),
+        )
+    )
+
+
 def _catalogue_declares_default_effort() -> bool:
     """Whether the loaded cost map carries default_reasoning_effort for ANY entry.
 
@@ -93,6 +114,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
 
     OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",))
     ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",))
+    LOOKS_UP_BASE_MODEL: ClassVar[bool] = False
 
     @classmethod
     def is_model_gpt_5_model(cls, model: str) -> bool:
@@ -307,7 +329,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
         model: str,
         drop_params: bool,
         *,
-        declared_reasoning_efforts: Collection[str] = (),
+        declared_reasoning_efforts: Collection[str] | None = None,
     ) -> dict:
         if self.is_model_gpt_5_search_model(model):
             if "max_tokens" in non_default_params:
@@ -338,10 +360,21 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
 
         if (
             requested_effort is not None
-            and requested_effort not in declared_reasoning_efforts
+            and requested_effort not in (declared_reasoning_efforts or ())
             and self.refuses_reasoning_effort_level(model, requested_effort)
         ):
             if litellm.drop_params or drop_params:
+                verbose_logger.warning(
+                    refused_reasoning_effort_warning(
+                        suggest_declared_levels=declared_reasoning_efforts is not None,
+                        suggest_base_model=self.LOOKS_UP_BASE_MODEL
+                        and not has_catalog_entry(self._model_map_lookup_name(model), None),
+                    ),
+                    "reasoning_effort",
+                    requested_effort,
+                    model,
+                    self._model_map_lookup_name(model),
+                )
                 non_default_params.pop("reasoning_effort", None)
                 optional_params.pop("reasoning_effort", None)
             else: