Merge remote-tracking branch 'origin/main' into litellm_kyma_provider

# Conflicts:
#	litellm/provider_endpoints_support_backup.json
#	provider_endpoints_support.json

.github/CODEOWNERS

@@ -1,5 +1,6 @@
 /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
+/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /helm/ @yassin-berriai
 /terraform/ @yassin-berriai
 /.github/workflows/zizmor.yml @yuneng-berri

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -84,9 +84,10 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 ## Screenshots / Proof of Fix
 
 <!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
-     The proof must be completely e2e with no mocks, using actual LLM calls costing real $$$ if applicable. `pytest` commands are not enough
-     Show ONLY the latest run: capture Before at the merge base and After at the PR's current tip, and when new commits change behavior, replace this whole section with the fresh run instead of stacking it on top of older ones. The run must be up to date. As soon as a new commit is made and it makes this PR description's after sha stale (it's no longer tip of PR), you must re-run the QA
-     Structure the section exactly as below: Before and After one heading level below this section, each naming the commit hash it was captured at, one lower-level heading per case inside each, the same case names in the same order on both sides, and numbered steps (command, observed output) under every case, never loose prose; shared setup (config, payloads) goes above Before, and with a single case, drop the case headings and number the steps directly
+     Simple PR exception (2026-10-28, #49030). A mechanical change fully proven by focused checks may skip before/after e2e. Follow AGENTS.md's eligibility rule, trace callers, and verify bug-fix regression sensitivity. Record E2E exception at <tip SHA>, scope, rationale, commands and results here. Tests/static checks suffice only for this exception. Keep CI and bot gates, reassess changed diffs, and honor explicit e2e requests. Omit the Before/After scaffold for an exception
+     Otherwise proof must be fully e2e, no mocks, with real provider calls where applicable. Pytest alone is insufficient. The following format applies to live proof
+     Show only the latest proof. Before uses the merge base and After the tested tip. Rerun affected QA after behavior changes. Verified neutral diffs may carry proof with the tested SHA, current tip, equivalence reason and focused checks, following AGENTS.md and /qa
+     Use matching Before/After case headings with numbered commands and observed output. Put shared setup above them. For one case, omit case headings
 
 ### Before (<hash>)
 

.github/workflows/cost-map-guard.yml

@@ -21,16 +21,18 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.base.sha }}
           persist-credentials: false
-      - name: Fetch the pull request head and its merge base
+      - name: Fetch the pull request head, its merge base, and the base branch tip
         id: revisions
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
-          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+          target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
+          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
           echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
+          echo "target=$target" >> "$GITHUB_OUTPUT"
       - name: Set up uv
         uses: ./.github/actions/setup-uv-with-retries
         with:
@@ -38,6 +40,8 @@ jobs:
       - name: Run the guard
         env:
           MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
+          COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }}
+          COST_MAP_GUARD_AUTHOR_TYPE: ${{ github.event.pull_request.user.type }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           HEAD_REF: ${{ github.event.pull_request.head.ref }}
         run: |

.github/workflows/cost-map-pins-recheck.yml

@@ -0,0 +1,91 @@
+name: Cost map pins recheck
+
+on:
+  push:
+    branches:
+      - main
+      - "litellm_**"
+    paths:
+      - model_prices_and_context_window.json
+      - litellm/model_prices_and_context_window_backup.json
+      - ci_cd/cost_map_pins.json
+
+permissions: {}
+
+jobs:
+  rerun-cost-map-guard:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      actions: write
+      contents: read
+      pull-requests: read
+    steps:
+      - name: Re-run the cost map guard on open pull requests the push could break
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_REF: ${{ github.ref_name }}
+          BEFORE_SHA: ${{ github.event.before }}
+          AFTER_SHA: ${{ github.event.after }}
+          FORCED: ${{ github.event.forced }}
+          COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]'
+          PINS_PATHS: '["ci_cd/cost_map_pins.json"]'
+        run: |
+          if [[ "$BEFORE_SHA" =~ ^0+$ || "$FORCED" == true ]] \
+            || ! gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${AFTER_SHA}?per_page=1" \
+              --jq '(env.PINS_PATHS | fromjson) as $pins | (.files | length) < 300 and all(.files[]; [.filename, .previous_filename] | any(IN($pins[])) | not)' \
+              | grep -qx true; then
+            export RECHECK_PATHS="$COST_MAP_PATHS"
+          else
+            export RECHECK_PATHS="$PINS_PATHS"
+          fi
+          rerun_cutoff="$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ)"
+          candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \
+            -f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) {
+              repository(owner: $owner, name: $name) {
+                pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) {
+                  nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } }
+                  pageInfo { hasNextPage endCursor }
+                }
+              }
+            }' \
+            --jq '(env.RECHECK_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[]
+              | {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)}
+              | select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')"
+          failed=0
+          declare -A rerun_heads
+          while read -r number head_sha listed; do
+            [ -n "$number" ] || continue
+            if [ "$listed" = false ]; then
+              if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \
+                --jq '(env.RECHECK_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then
+                echo "::error::could not list the files of #$number"
+                failed=1
+                continue
+              fi
+              [ -n "$changed" ] || continue
+            fi
+            [ -z "${rerun_heads[$head_sha]:-}" ] || continue
+            rerun_heads[$head_sha]=1
+            if ! runs="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId,createdAt --jq '.[] | "\(.databaseId) \(.createdAt)"')"; then
+              echo "::error::could not list the cost map guard runs of #$number"
+              failed=1
+              continue
+            fi
+            if [ -z "$runs" ]; then
+              echo "::warning::no cost map guard run found for #$number"
+              continue
+            fi
+            while read -r run_id created_at; do
+              if [[ "$created_at" < "$rerun_cutoff" ]]; then
+                echo "::warning::cost map guard run $run_id for #$number is over 30 days old, so GitHub cannot re-run it; #$number gets the pin checks on its next push"
+                continue
+              fi
+              gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true
+              if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then
+                echo "::error::could not re-run cost map guard run $run_id for #$number"
+                failed=1
+              fi
+            done <<< "$runs"
+          done <<< "$candidates"
+          exit "$failed"

.github/workflows/test-unit.yml

@@ -236,6 +236,7 @@ jobs:
               tests/unit/llms/azure_ai
               tests/unit/llms/base_llm
               tests/unit/llms/baseten
+              tests/unit/llms/bigmodel
               tests/unit/llms/black_forest_labs
               tests/unit/llms/brave
               tests/unit/llms/bytez

.grype.yaml

@@ -11,3 +11,12 @@ ignore:
     package:
       name: zlib
       type: apk
+  - vulnerability: GHSA-v4x8-3wr5-3wq8
+    reason: >-
+      filelock ReadWriteLock advisory, and no package in the image takes ReadWriteLock;
+      the fix (4.2.1) is blocked by litellm's own filelock<4.0 cap until #48635 or #48802 lifts it,
+      and the image then moves off 3.32.7, which retires this version-scoped rule
+    package:
+      name: filelock
+      version: 3.32.7
+      type: python

AGENTS.md

@@ -41,7 +41,9 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
 
-Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
+Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
+
+For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce
 
 If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
 - don't use emojis
@@ -107,6 +109,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - No file sprawl: deliberate file and folder structure
 - Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
 - API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
+- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
 
 Follow conventional commits for commit names and PR titles
 

ci_cd/cost_map_guard.py

@@ -1,37 +1,67 @@
 """Guard the cost map on pull requests.
 
-Every pull request whose diff against its merge base touches one of the two cost map files gets the file
-checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
-skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
-branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
-get the file checks and additionally may only touch those two files and may only add or update models.
+Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the
+file checks: the files parse, the backup copy matches the root file, and every capability pinned in
+ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says
+otherwise) still holds in the tree that merging the pull request into its base branch produces, so a branch cut before
+a pin was added cannot lower it, and a pull request that deletes the pins file fails too. A pull request that leaves
+all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries
+whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named
+litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may
+only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests
+(branches named litellm-providers/price-sync*) and pull requests a bot account opened (COST_MAP_GUARD_AUTHOR_TYPE is
+Bot) get the human checks and may not touch the pins file either.
 """
 
 from __future__ import annotations
 
 import argparse
 import json
+import os
 import subprocess
 import sys
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final
 
 COST_MAP_PATH: Final = "model_prices_and_context_window.json"
 BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
+PINS_PATH: Final = "ci_cd/cost_map_pins.json"
 GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
+CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH)
 BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync"
+TARGET_ENV: Final = "COST_MAP_GUARD_TARGET"
+AUTHOR_TYPE_ENV: Final = "COST_MAP_GUARD_AUTHOR_TYPE"
+BOT_AUTHOR_TYPE: Final = "Bot"
+MERGE_CONFLICT_EXIT_CODE: Final = 1
 SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
 EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
 
 CostMap = dict[str, object]
+_MISSING: Final = object()
 
 
 @dataclass(frozen=True, slots=True)
 class Snapshot:
     cost_map: str
     backup: str
+    pins: str
+
+
+@dataclass(frozen=True, slots=True)
+class MergeFailed:
+    stderr: str
+
+
+@dataclass(frozen=True, slots=True)
+class Pin:
+    key: str
+    field: str
+    value: object
+    checked: str
+    source: str
 
 
 def _parse_object(text: str, path: str) -> CostMap | str:
@@ -91,6 +121,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
     return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
 
 
+def _pin(key: str, field: str, spec: object) -> Pin | str:
+    malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source"
+    if not isinstance(spec, dict) or "value" not in spec:
+        return malformed
+    checked: Final = spec.get("checked")
+    source: Final = spec.get("source")
+    if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source:
+        return malformed
+    return Pin(key, field, spec["value"], checked, source)
+
+
+def _pins(parsed: CostMap) -> Iterator[Pin | str]:
+    for key, fields in parsed.items():
+        if not isinstance(fields, dict):
+            yield f"{PINS_PATH}: {key} must map field names to pins"
+            continue
+        for field, spec in fields.items():
+            yield _pin(key, field, spec)
+
+
+def _pin_violations(parsed: CostMap, entries: Mapping[str, Mapping[str, object]]) -> Iterator[str]:
+    for pin in _pins(parsed):
+        if isinstance(pin, str):
+            yield pin
+        elif pin.key not in entries:
+            yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry"
+        elif entries[pin.key].get(pin.field, _MISSING) != pin.value:
+            yield (
+                f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} "
+                f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first"
+            )
+
+
+def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]:
+    if not pins_text:
+        return (f"{PINS_PATH} is missing; restore it, its pins were verified against live provider calls",)
+    parsed: Final = _parse_object(pins_text, PINS_PATH)
+    if isinstance(parsed, str):
+        return (parsed,)
+    return tuple(_pin_violations(parsed, _entries(head_map)))
+
+
 def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
     base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
     if isinstance(base_map, str):
@@ -120,8 +192,18 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
     )
 
 
+def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]:
+    if PINS_PATH not in changed_files:
+        return ()
+    if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX):
+        return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
+    if author_type == BOT_AUTHOR_TYPE:
+        return (f"PRs opened by a bot may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
+    return ()
+
+
 def touches_cost_map(changed_files: Sequence[str]) -> bool:
-    return any(path in GUARDED_PATHS for path in changed_files)
+    return any(path in CHECKED_PATHS for path in changed_files)
 
 
 def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
@@ -130,9 +212,21 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
     return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
 
 
+def _merged_pin_failures(base: Snapshot, merged: Snapshot | None) -> tuple[str, ...]:
+    if merged is None:
+        return ("the pull request does not merge cleanly into its base branch; merge the base branch in first",)
+    if not merged.pins and not base.pins:
+        return ()
+    merged_map: Final = _parse_object(merged.cost_map, COST_MAP_PATH)
+    if isinstance(merged_map, str):
+        return (f"after merging into the base branch, {merged_map}",)
+    return pin_failures(merged.pins, merged_map)
+
+
 def guard_failures(
     base: Snapshot,
     head: Snapshot,
+    merged: Snapshot | None,
     changed_files: Sequence[str],
     bot: bool,
     *,
@@ -147,6 +241,7 @@ def guard_failures(
         *_file_failures(
             head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
         ),
+        *_merged_pin_failures(base, merged),
         *(_bot_failures(base, head_map, changed_files) if bot else ()),
     )
 
@@ -157,7 +252,22 @@ def _git(*args: str) -> str | None:
 
 
 def snapshot(revision: str) -> Snapshot:
-    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
+    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS))
+
+
+def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None:
+    result: Final = subprocess.run(
+        ("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head),
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    tree: Final = result.stdout.split("\n", 1)[0]
+    if result.returncode == 0:
+        return tree
+    if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree:
+        return None
+    return MergeFailed(result.stderr.strip())
 
 
 def changed_files(base: str, head: str) -> tuple[str, ...] | None:
@@ -169,6 +279,13 @@ def main(argv: Sequence[str]) -> int:
     parser: Final = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--base", required=True, help="merge base of the pull request")
     parser.add_argument("--head", required=True, help="head commit of the pull request")
+    env_target: Final = os.environ.get(TARGET_ENV)
+    parser.add_argument(
+        "--target",
+        default=env_target,
+        required=env_target is None,
+        help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}",
+    )
     parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
     args: Final = parser.parse_args(argv)
     bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
@@ -176,7 +293,19 @@ def main(argv: Sequence[str]) -> int:
     if changed is None:
         print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
         return 1
-    failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
+    if not bot and not touches_cost_map(changed):
+        print(f"cost map guard passed ({contract_for(bot, changed)})")
+        return 0
+    merged: Final = merged_tree(args.base, args.target, args.head)
+    if isinstance(merged, MergeFailed):
+        print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}")
+        return 1
+    failures: Final = (
+        *guard_failures(
+            snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot
+        ),
+        *pin_author_failures(args.head_ref, os.environ.get(AUTHOR_TYPE_ENV, ""), changed),
+    )
     contract: Final = contract_for(bot, changed)
     if failures:
         print(f"cost map guard failed ({contract}):")

ci_cd/cost_map_pins.json

@@ -0,0 +1,16 @@
+{
+    "fireworks_ai/accounts/fireworks/models/minimax-m3": {
+        "supports_vision": {
+            "value": true,
+            "checked": "2026-09-26",
+            "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
+        }
+    },
+    "fireworks_ai/minimax-m3": {
+        "supports_vision": {
+            "value": true,
+            "checked": "2026-09-26",
+            "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
+        }
+    }
+}

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Mapping
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, cast
+from typing import TYPE_CHECKING, Final
 
 from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
     ManagedObjectCreatorAttribution,
@@ -290,12 +290,14 @@ class CheckResponsesCost:
         sees provider env vars, so it fails for every deployment whose credentials
         live in the config; the row then never leaves ``queued``.
         """
-        if not via_router:
-            return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
-        router_response = await self.llm_router.aget_responses(
-            response_id=response_id, litellm_metadata=litellm_metadata
+        retrieved: Final[object] = (
+            await self.llm_router.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
+            if via_router
+            else await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
         )
-        return cast(ResponsesAPIResponse, router_response)
+        if not isinstance(retrieved, ResponsesAPIResponse):
+            raise TypeError(f"Retrieving {response_id} did not return a response object")
+        return retrieved
 
     async def _fetch_unbilled(self, target: _PollTarget) -> ResponsesAPIResponse:
         return await self._get_response(

examples/bigmodel.md

@@ -0,0 +1,63 @@
+# BigModel (Zhipu AI)
+
+Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL
+
+```yaml
+model_list:
+  - model_name: glm
+    litellm_params:
+      model: bigmodel/glm-5.3
+      api_key: os.environ/BIGMODEL_API_KEY
+      api_base: https://open.bigmodel.cn
+```
+
+One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API
+
+| Proxy endpoint | BigModel endpoint |
+| --- | --- |
+| `/v1/chat/completions` | `/api/paas/v4/chat/completions` |
+| `/v1/responses` | `/api/v1/responses` |
+| `/v1/messages` | `/api/anthropic/v1/messages` |
+
+`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix
+
+A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above
+
+The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY`
+
+```python
+import litellm
+
+chat = litellm.completion(
+    model="bigmodel/glm-5.3",
+    messages=[{"role": "user", "content": "Hello"}],
+    thinking={"type": "enabled"},
+    reasoning_effort="low",
+)
+
+response = litellm.responses(
+    model="bigmodel/glm-5.3",
+    input="Hello",
+    reasoning={"effort": "low"},
+    store=True,
+)
+
+async def messages():
+    return await litellm.anthropic.messages.acreate(
+        model="bigmodel/glm-5.3",
+        max_tokens=1024,
+        messages=[{"role": "user", "content": "Hello"}],
+        thinking={"type": "adaptive"},
+        output_config={"effort": "low"},
+    )
+```
+
+Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route
+
+GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters
+
+Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search
+
+The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices
+
+API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27

litellm/__init__.py

@@ -39,6 +39,7 @@ from litellm._logging import (
     turn_on_json,
     log_level,
 )
+from litellm._logging import suppress_startup_output as suppress_startup_output
 import re
 from litellm.constants import (
     DEFAULT_BATCH_SIZE,
@@ -591,7 +592,7 @@ vertex_ai_safety_settings: Optional[dict] = None
 ####### COMPLETION MODELS ###################
 from typing import Set
 
-open_ai_chat_completion_models: Set = set()
+open_ai_chat_completion_models: Set[str] = set()
 open_ai_text_completion_models: Set = set()
 cohere_models: Set = set()
 cohere_chat_models: Set = set()
@@ -619,6 +620,8 @@ vertex_openai_models: Set = set()
 vertex_minimax_models: Set = set()
 vertex_moonshot_models: Set = set()
 vertex_zai_models: Set = set()
+vertex_qwen_models: Final[Set[str]] = set()
+vertex_ai_models: Final[Set[str]] = set()
 ai21_models: Set = set()
 ai21_chat_models: Set = set()
 nlp_cloud_models: Set = set()
@@ -635,6 +638,7 @@ watsonx_models: Set = set()
 gemini_models: Set = set()
 xai_models: Set = set()
 zai_models: Set = set()
+bigmodel_models: Final[set[str]] = set()  # mutable-ok: provider registry is populated and extended at runtime
 deepseek_models: Set = set()
 tencent_models: Set = set()
 runwayml_models: Set = set()
@@ -834,6 +838,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
         elif value.get("litellm_provider") == "vertex_ai-zai_models":
             key = key.replace("vertex_ai/", "")
             vertex_zai_models.add(key)
+        elif value.get("litellm_provider") == "vertex_ai-qwen_models":
+            vertex_qwen_models.add(key.removeprefix("vertex_ai/"))
+        elif value.get("litellm_provider") == "vertex_ai":
+            vertex_ai_models.add(key.removeprefix("vertex_ai/"))
         elif value.get("litellm_provider") == "ai21":
             if value.get("mode") == "chat":
                 ai21_chat_models.add(key)
@@ -878,6 +886,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             xai_models.add(key)
         elif value.get("litellm_provider") == "zai":
             zai_models.add(key)
+        elif value.get("litellm_provider") == "bigmodel":
+            bigmodel_models.add(key)
         elif value.get("litellm_provider") == "fal_ai":
             fal_ai_models.add(key)
         elif value.get("litellm_provider") == "deepseek":
@@ -1110,6 +1120,7 @@ model_list = list(
     | text_completion_inception_models
     | xai_models
     | zai_models
+    | bigmodel_models
     | fal_ai_models
     | deepseek_models
     | modelscope_models
@@ -1208,7 +1219,16 @@ def _build_models_by_provider() -> dict:
         | vertex_deepseek_models
         | vertex_minimax_models
         | vertex_moonshot_models
-        | vertex_zai_models,
+        | vertex_zai_models
+        | vertex_qwen_models
+        | vertex_mistral_models
+        | vertex_llama3_models
+        | vertex_ai_ai21_models
+        | vertex_openai_models
+        | vertex_ai_image_models
+        | vertex_ai_video_models
+        | vertex_embedding_models
+        | vertex_ai_models,
         "ai21": ai21_models | ai21_chat_models,
         "bedrock": bedrock_models | bedrock_converse_models,
         "petals": petals_models,
@@ -1225,6 +1245,7 @@ def _build_models_by_provider() -> dict:
         "text-completion-inception": text_completion_inception_models,
         "xai": xai_models,
         "zai": zai_models,
+        "bigmodel": bigmodel_models,
         "fal_ai": fal_ai_models,
         "deepseek": deepseek_models,
         "tencent": tencent_models,

litellm/_logging.py

@@ -10,10 +10,12 @@ import sys
 from collections.abc import Iterator, Mapping
 from datetime import datetime
 from logging import Formatter
+from types import FrameType
 from typing import Final, TextIO
 from urllib.parse import unquote
 
 from pydantic import TypeAdapter
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm.constants import (
@@ -598,6 +600,7 @@ _LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s"
 _LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s"
 _PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX
 _COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}"
+_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m")
 
 
 def _stream_is_tty(stream: TextIO | None) -> bool:
@@ -612,15 +615,28 @@ def _stream_is_tty(stream: TextIO | None) -> bool:
         return False
 
 
-def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
-    """The plain-text log format, colorized only when both streams are an interactive terminal.
+def _color_enabled(stream: TextIO | None) -> bool:
+    """True when ANSI colors written to the stream reach an interactive terminal.
 
     Honors the NO_COLOR convention from no-color.org: color is disabled when
     NO_COLOR is present with a non-empty value.
     """
-    if os.environ.get("NO_COLOR"):
-        return _PLAIN_LOG_FORMAT
-    return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT
+    return not os.environ.get("NO_COLOR") and _stream_is_tty(stream)
+
+
+def _terminal_colors(stdout: TextIO | None, stderr: TextIO | None) -> bool:
+    """True when both streams are an interactive color terminal, the one rule for log lines and startup prints."""
+    return _color_enabled(stdout) and _color_enabled(stderr)
+
+
+def terminal_text(message: str) -> str:
+    """The message with its ANSI colors kept only on a color terminal, by the rule the plain log format uses."""
+    return message if _terminal_colors(sys.stdout, sys.stderr) else _ANSI_ESCAPE.sub("", message)
+
+
+def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
+    """The plain-text log format, colorized only when both streams are an interactive terminal."""
+    return _COLOR_LOG_FORMAT if _terminal_colors(stdout, stderr) else _PLAIN_LOG_FORMAT
 
 
 class LevelRoutingStreamHandler(logging.StreamHandler):
@@ -642,8 +658,8 @@ class LevelRoutingStreamHandler(logging.StreamHandler):
         super().emit(record)
 
 
-def _parse_json_logs_env(value: str | None) -> bool:
-    """Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs.
+def _parse_env_flag(value: str | None) -> bool:
+    """Strict opt-in parse for a boolean env var such as JSON_LOGS: only "true" (any case) enables it.
 
     Matches the reader in litellm-proxy-extras/_logging.py. The previous
     bool(os.getenv(...)) treated any non-empty value, including "false" and "0",
@@ -656,7 +672,18 @@ def resolve_log_level(log_level: str) -> int:
     return getattr(logging, log_level.upper())
 
 
-json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS"))
+json_logs: Final = _parse_env_flag(os.getenv("JSON_LOGS"))
+suppress_startup_output: Final = _parse_env_flag(os.getenv("LITELLM_SUPPRESS_STARTUP_OUTPUT"))
+
+
+class StartupOutputSettings(TypedDict, total=False):
+    json_logs: ReadOnly[object]
+    suppress_startup_output: ReadOnly[object]
+
+
+STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(StartupOutputSettings)
+
+
 # Create a handler for the logger (you may need to adapt this based on your needs)
 log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
 numeric_level: Final[int] = resolve_log_level(log_level)
@@ -743,7 +770,7 @@ class JsonFormatter(Formatter):
         return dt.isoformat()
 
     def format(self, record):
-        message_str: Final = record.getMessage()
+        message_str: Final = _ANSI_ESCAPE.sub("", record.getMessage())
         json_record: Final[dict[str, object]] = {
             "message": message_str,
             "level": record.levelname,
@@ -799,12 +826,14 @@ class CorrelationPlainFormatter(logging.Formatter):
     """Appends trace_id/session_id to plain-text log lines stamped by CorrelationContextFilter.
 
     Mirrors JsonFormatter's handling of these two fields so request_correlation_in_logs
-    behaves the same whether or not json_logs is enabled.
+    behaves the same whether or not json_logs is enabled. Unless it was built with the color format,
+    it also drops the ANSI color codes a message carries, which a piped log would store as text.
     """
 
     def format(self, record: logging.LogRecord) -> str:
         rendered: Final = super().format(record)
-        formatted: Final = rendered if _is_redacted(record) else redact_string(rendered)
+        uncolored: Final = rendered if self._fmt == _COLOR_LOG_FORMAT else _ANSI_ESCAPE.sub("", rendered)
+        formatted: Final = uncolored if _is_redacted(record) else redact_string(uncolored)
         trace_id: Final = getattr(record, "trace_id", None)
         session_id: Final = getattr(record, "session_id", None)
         if not trace_id and not session_id:
@@ -898,29 +927,46 @@ verbose_proxy_logger.addFilter(_diagnostic_filter)
 verbose_proxy_stdout_logger.addFilter(_diagnostic_filter)
 verbose_logger.addFilter(_diagnostic_filter)
 
-_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m")
-
 
 def print_startup_message(message: str) -> None:
     """Write a boot-time message meant for the operator's terminal.
 
-    Plain mode prints it unchanged, colors and spacing included. With JSON logs on, every line of
-    stdout is read by a log pipeline, so the message goes out as one INFO record of the proxy logger
-    instead, on stdout like the print whatever handlers the logging config gave that logger, with the
-    ANSI color codes and surrounding blank lines removed, located at the caller and emitted at any log
-    level, whole and unredacted as the print showed it; a blank spacer produces no record.
+    LITELLM_SUPPRESS_STARTUP_OUTPUT or litellm_settings.suppress_startup_output turns every one of them
+    off; a warning goes through print_startup_warning instead. Plain mode prints it with its spacing,
+    keeping the ANSI colors only on a color terminal by the same check as the plain log format, so a
+    container log never stores escape codes.
+    With JSON logs on, every line of stdout is read by a log pipeline, so the message goes out as one
+    INFO record of the proxy logger instead, on stdout like the print whatever handlers the logging
+    config gave that logger, with the ANSI color codes and surrounding blank lines removed, located at
+    the caller and emitted at any log level, whole and unredacted as the print showed it; a blank
+    spacer produces no record.
     """
+    if litellm.suppress_startup_output is True:
+        return
+    frame: Final = inspect.currentframe()
+    _write_startup_message(message, frame.f_back if frame is not None else None, logging.INFO)
+
+
+def print_startup_warning(message: str) -> None:
+    """Write a boot-time warning as print_startup_message writes a message, even with suppress_startup_output
+    on, since it names something the operator has to fix. With JSON logs on it is a WARNING record, on
+    stdout where the print put it, so a collector that filters on the level field keeps it."""
+    frame: Final = inspect.currentframe()
+    _write_startup_message(message, frame.f_back if frame is not None else None, logging.WARNING)
+
+
+def _write_startup_message(message: str, caller: FrameType | None, level: int) -> None:
     if not litellm.json_logs:
-        print(message)  # noqa: T201  # plain mode: this is the operator-terminal print it replaces
+        import click
+
+        click.echo(terminal_text(message))
         return
     text: Final = _ANSI_ESCAPE.sub("", message).strip("\n")
     if not text:
         return
-    frame: Final = inspect.currentframe()
-    caller: Final = frame.f_back if frame is not None else None
     record: Final = verbose_proxy_logger.makeRecord(
         verbose_proxy_logger.name,
-        logging.INFO,
+        level,
         caller.f_code.co_filename if caller is not None else __file__,
         caller.f_lineno if caller is not None else 0,
         text,
@@ -934,8 +980,8 @@ def print_startup_message(message: str) -> None:
 
 
 def _startup_json_handler() -> logging.Handler:
-    """The proxy's JSON handler on stdout, filters included, for one startup message."""
-    handler: Final = LevelRoutingStreamHandler()
+    """The proxy's JSON handler on stdout, filters included, for one startup message or warning."""
+    handler: Final = logging.StreamHandler(sys.stdout)
     handler.setFormatter(JsonFormatter())
     handler.addFilter(_secret_filter)
     handler.addFilter(_correlation_filter)

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -260,6 +260,7 @@ class ResponsesToCompletionBridgeHandler:
                 structured_output=is_structured_output_request(
                     cast("Mapping[str, object]", optional_params)  # cast-ok: optional_params is an untyped dict
                 ),
+                replays_openai_history=custom_llm_provider == "openai",
             )
             streamwrapper: Final = CustomStreamWrapper(
                 completion_stream=completion_stream,
@@ -363,6 +364,7 @@ class ResponsesToCompletionBridgeHandler:
                 structured_output=is_structured_output_request(
                     cast("Mapping[str, object]", optional_params)  # cast-ok: optional_params is an untyped dict
                 ),
+                replays_openai_history=custom_llm_provider == "openai",
             )
             streamwrapper: Final = CustomStreamWrapper(
                 completion_stream=completion_stream,

litellm/completion_extras/litellm_responses_transformation/history.py

@@ -0,0 +1,432 @@
+import hashlib
+import json
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from itertools import accumulate, chain, pairwise
+from types import MappingProxyType
+from typing import Annotated, Final, Literal, TypeAlias
+
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
+
+REPLAY_FIELD: Final = "openai_responses_replay"
+PHASE_FIELD: Final = "phase"
+
+Phase: TypeAlias = Literal["commentary", "final_answer"]
+
+
+class _Frozen(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+
+class _OutputText(_Frozen):
+    type: Literal["output_text"]
+    text: str
+
+
+class _OutputMessage(_Frozen):
+    type: Literal["message"]
+    role: Literal["assistant"]
+    content: tuple[_OutputText, ...]
+    phase: Phase | None = None
+
+
+class _OutputReasoning(_Frozen):
+    type: Literal["reasoning"]
+    id: str
+
+
+class _OutputToolCall(_Frozen):
+    type: Literal["function_call", "custom_tool_call"]
+    call_id: str
+    id: str | None = None
+
+
+class _OutputToolSearch(_Frozen):
+    type: Literal["tool_search_call", "tool_search_output"]
+
+
+_OutputItem: TypeAlias = Annotated[
+    _OutputMessage | _OutputReasoning | _OutputToolCall | _OutputToolSearch, Field(discriminator="type")
+]
+_OUTPUT_ITEM: Final[TypeAdapter[_OutputItem]] = TypeAdapter(_OutputItem)
+
+
+class _AnyMessage(_Frozen):
+    type: Literal["message"]
+
+
+class _MessageEntry(_Frozen):
+    type: Literal["message"]
+    parts: tuple[Annotated[int, Field(ge=0)], ...]
+    phase: Phase | None = None
+
+
+class _IndexedEntry(_Frozen):
+    type: Literal["reasoning", "function_call", "custom_tool_call", "tool_search_call", "tool_search_output"]
+    index: Annotated[int, Field(ge=0)]
+
+
+_Entry: TypeAlias = Annotated[_MessageEntry | _IndexedEntry, Field(discriminator="type")]
+
+
+class _Replay(_Frozen):
+    version: Literal[1]
+    items: tuple[_Entry, ...]
+    checksum: str
+
+
+class _Namespaced(_Frozen):
+    namespace: str | None = None
+
+
+class _ToolFunction(_Frozen):
+    name: str = ""
+    arguments: str = ""
+    provider_specific_fields: _Namespaced | None = None
+
+
+class _ToolCall(_Frozen):
+    id: str
+    function: _ToolFunction
+    provider_specific_fields: _Namespaced | None = None
+
+
+class _SummaryText(_Frozen):
+    type: str = "summary_text"
+    text: str = ""
+
+
+class _Reasoning(_Frozen):
+    id: str | None = None
+    encrypted_content: str | None = None
+    summary: tuple[_SummaryText, ...] | None = None
+
+
+class _ProviderFields(_Frozen):
+    phase: Phase | None = None
+    tool_search_items: tuple[Mapping[str, object], ...] | None = None
+    openai_responses_replay: object = None
+
+
+class _ChatMessage(_Frozen):
+    role: str | None = None
+    content: str | None = None
+    tool_calls: tuple[_ToolCall, ...] | None = None
+    reasoning_items: tuple[_Reasoning, ...] | None = None
+    phase: Phase | None = None
+    provider_specific_fields: _ProviderFields | None = None
+
+
+class _PhaseFields(_Frozen):
+    phase: Phase | None = None
+
+
+class _RequestedPhaseFields(_Frozen):
+    phase: object = None
+
+
+class _RequestedPhase(_Frozen):
+    phase: object = None
+    provider_specific_fields: _RequestedPhaseFields | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class UnknownPhase:
+    value: object
+
+
+_PHASE: Final[TypeAdapter[Phase]] = TypeAdapter(Phase)
+
+
+class _RetainedFields(_PhaseFields):
+    openai_responses_replay: object = None
+
+
+@dataclass(frozen=True, slots=True)
+class ReplayedTurn:
+    items: tuple[Mapping[str, object], ...]
+    custom_call_ids: frozenset[str]
+
+
+def _as_json(value: object) -> object:
+    return value.model_dump() if isinstance(value, BaseModel) else value
+
+
+def _parsed_output_item(item: object) -> _OutputItem | None:
+    try:
+        return _OUTPUT_ITEM.validate_python(_as_json(item))
+    except ValidationError:
+        return None
+
+
+def _is_message(item: object) -> bool:
+    try:
+        _AnyMessage.model_validate(_as_json(item))
+    except ValidationError:
+        return False
+    return True
+
+
+def _chat_message(message: object) -> _ChatMessage | None:
+    try:
+        return _ChatMessage.model_validate(_as_json(message))
+    except ValidationError:
+        return None
+
+
+def requested_phase(message: object) -> Phase | UnknownPhase | None:
+    try:
+        requested: Final = _RequestedPhase.model_validate(_as_json(message))
+    except ValidationError:
+        return None
+    fields: Final = requested.provider_specific_fields
+    value: Final = requested.phase or (fields.phase if fields else None)
+    if value is None:
+        return None
+    try:
+        return _PHASE.validate_python(value)
+    except ValidationError:
+        return UnknownPhase(value)
+
+
+def reasoning_input_item(
+    item_id: str | None, summary: Sequence[object] | None, encrypted_content: str | None
+) -> Mapping[str, object]:
+    return {
+        "type": "reasoning",
+        **({"id": item_id} if item_id else {}),
+        "summary": list(summary or ()),
+        **({"encrypted_content": encrypted_content} if encrypted_content else {}),
+    }
+
+
+def retained_replay_fields(provider_specific_fields: object) -> Mapping[str, object] | None:
+    try:
+        retained: Final = _RetainedFields.model_validate(provider_specific_fields)
+    except ValidationError:
+        return None
+    return retained.model_dump(exclude_none=True) or None
+
+
+def output_message_text(output_items: Sequence[object], *, first_message_only: bool) -> str:
+    parsed: Final = map(_parsed_output_item, output_items)
+    messages: Final = tuple(item for item in parsed if isinstance(item, _OutputMessage))
+    return "".join(_texts(messages[:1] if first_message_only else messages))
+
+
+def _tool_search_entries(message: _ChatMessage) -> tuple[Mapping[str, object], ...]:
+    fields: Final = message.provider_specific_fields
+    items: Final = (fields.tool_search_items if fields else None) or ()
+    return tuple(item for item in items if item.get("type") in ("tool_search_call", "tool_search_output"))
+
+
+def _checksum(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> str:
+    fields: Final = message.provider_specific_fields
+    bound: Final = {
+        "content": message.content or "",
+        "tool_calls": [[call.id, call.function.name, call.function.arguments] for call in message.tool_calls or ()],
+        "reasoning": [
+            [item.id, item.encrypted_content, [part.text for part in item.summary or ()]]
+            for item in message.reasoning_items or ()
+        ],
+        "tool_search_items": list((fields.tool_search_items if fields else None) or ()),
+        "phase": message.phase or (fields.phase if fields else None),
+        "items": [entry.model_dump(mode="json", exclude_none=True) for entry in entries],
+    }
+    encoded: Final = json.dumps(bound, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return hashlib.sha256(encoded.encode("utf-8", "surrogatepass")).hexdigest()
+
+
+def _kind(item: _OutputItem) -> str:
+    match item:
+        case _OutputToolCall():
+            return "tool_call"
+        case _OutputToolSearch():
+            return "tool_search"
+        case _OutputMessage() | _OutputReasoning():
+            return item.type
+
+
+def _entry(items: Sequence[_OutputItem], position: int) -> _MessageEntry | _IndexedEntry:
+    item: Final = items[position]
+    if isinstance(item, _OutputMessage):
+        return _MessageEntry(type="message", parts=tuple(len(part.text) for part in item.content), phase=item.phase)
+    ordinal: Final = sum(1 for earlier in items[:position] if _kind(earlier) == _kind(item))
+    return _IndexedEntry(type=item.type, index=ordinal)
+
+
+def _texts(items: Sequence[_OutputItem]) -> Iterator[str]:
+    for item in items:
+        if isinstance(item, _OutputMessage):
+            yield from (part.text for part in item.content)
+
+
+def _matches(items: Sequence[_OutputItem], message: _ChatMessage) -> bool:
+    reasoning_ids: Final = tuple(item.id for item in items if isinstance(item, _OutputReasoning))
+    calls: Final = tuple(item for item in items if isinstance(item, _OutputToolCall))
+    tool_calls: Final = message.tool_calls or ()
+    searches: Final = sum(1 for item in items if isinstance(item, _OutputToolSearch))
+    return (
+        "".join(_texts(items)) == (message.content or "")
+        and reasoning_ids == tuple(item.id for item in message.reasoning_items or ())
+        and len(calls) == len(tool_calls)
+        and all(chat.id in (call.call_id, call.id) for call, chat in zip(calls, tool_calls))
+        and searches == len(_tool_search_entries(message))
+    )
+
+
+def _content_phase(
+    output_items: Sequence[object], parsed: Sequence[_OutputItem | None], *, first_message_only: bool
+) -> Phase | None:
+    messages: Final = tuple(item for raw, item in zip(output_items, parsed) if _is_message(raw))
+    if not messages:
+        return None
+    content_message: Final = messages[0] if first_message_only else messages[-1]
+    return content_message.phase if isinstance(content_message, _OutputMessage) else None
+
+
+def _with_phase(message: _ChatMessage, phase: Phase | None) -> _ChatMessage:
+    if phase is None:
+        return message
+    fields: Final = (message.provider_specific_fields or _ProviderFields()).model_copy(update={PHASE_FIELD: phase})
+    return message.model_copy(update={"provider_specific_fields": fields})
+
+
+def replay_fields(
+    output_items: Sequence[object], message: object, *, completed: bool, first_message_only: bool
+) -> Mapping[str, object]:
+    parsed: Final = tuple(map(_parsed_output_item, output_items))
+    phase: Final = _content_phase(output_items, parsed, first_message_only=first_message_only)
+    phase_fields: Final[Mapping[str, object]] = MappingProxyType({PHASE_FIELD: phase} if phase else {})
+    items: Final = tuple(item for item in parsed if item is not None)
+    parsed_message: Final = _chat_message(message)
+    chat: Final = _with_phase(parsed_message, phase) if parsed_message is not None else None
+    if not completed or not items or len(items) != len(parsed) or chat is None or not _matches(items, chat):
+        return phase_fields
+    entries: Final = tuple(_entry(items, position) for position in range(len(items)))
+    replay: Final = _Replay(version=1, items=entries, checksum=_checksum(chat, entries))
+    return MappingProxyType({**phase_fields, REPLAY_FIELD: replay.model_dump(mode="json", exclude_none=True)})
+
+
+def _stored_replay(message: _ChatMessage) -> _Replay | None:
+    fields: Final = message.provider_specific_fields
+    if fields is None or fields.openai_responses_replay is None:
+        return None
+    try:
+        return _Replay.model_validate(fields.openai_responses_replay)
+    except ValidationError:
+        return None
+
+
+def _message_texts(entries: Sequence[_MessageEntry], content: str) -> tuple[tuple[str, ...], ...]:
+    lengths: Final = tuple(chain.from_iterable(entry.parts for entry in entries))
+    bounds: Final = tuple(accumulate(lengths, initial=0))
+    parts: Final = tuple(content[start:end] for start, end in pairwise(bounds))
+    groups: Final = tuple(accumulate((len(entry.parts) for entry in entries), initial=0))
+    return tuple(parts[start:end] for start, end in pairwise(groups))
+
+
+def _message_item(texts: Sequence[str], phase: Phase | None) -> Mapping[str, object]:
+    return {
+        "type": "message",
+        "role": "assistant",
+        "content": [{"type": "output_text", "text": text} for text in texts],
+        **({PHASE_FIELD: phase} if phase else {}),
+    }
+
+
+def _function_call_item(call: _ToolCall) -> Mapping[str, object]:
+    namespaced: Final = call.provider_specific_fields or call.function.provider_specific_fields
+    namespace: Final = namespaced.namespace if namespaced else None
+    return {
+        "type": "function_call",
+        "call_id": call.id,
+        "name": call.function.name,
+        "arguments": call.function.arguments,
+        **({"namespace": namespace} if namespace else {}),
+    }
+
+
+def _custom_tool_call_item(call: _ToolCall) -> Mapping[str, object]:
+    return {
+        "type": "custom_tool_call",
+        "call_id": call.id,
+        "name": call.function.name,
+        "input": call.function.arguments,
+    }
+
+
+def _reasoning_item(item: _Reasoning) -> Mapping[str, object]:
+    summary: Final = tuple(part.model_dump() for part in item.summary or ())
+    return reasoning_input_item(item.id, summary, item.encrypted_content)
+
+
+@dataclass(frozen=True, slots=True)
+class _Resolved:
+    message: _ChatMessage
+    texts: Mapping[int, tuple[str, ...]]
+    tool_searches: tuple[Mapping[str, object], ...]
+
+
+def _input_item(resolved: _Resolved, position: int, entry: _MessageEntry | _IndexedEntry) -> Mapping[str, object]:
+    if isinstance(entry, _MessageEntry):
+        return _message_item(resolved.texts[position], entry.phase)
+    tool_calls: Final = resolved.message.tool_calls or ()
+    match entry.type:
+        case "reasoning":
+            return _reasoning_item((resolved.message.reasoning_items or ())[entry.index])
+        case "function_call":
+            return _function_call_item(tool_calls[entry.index])
+        case "custom_tool_call":
+            return _custom_tool_call_item(tool_calls[entry.index])
+        case "tool_search_call" | "tool_search_output":
+            return dict(resolved.tool_searches[entry.index])
+
+
+def _fits(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> bool:
+    tool_calls: Final = len(message.tool_calls or ())
+    tool_searches: Final = len(_tool_search_entries(message))
+    sizes: Final = MappingProxyType(
+        {
+            "reasoning": len(message.reasoning_items or ()),
+            "function_call": tool_calls,
+            "custom_tool_call": tool_calls,
+            "tool_search_call": tool_searches,
+            "tool_search_output": tool_searches,
+        }
+    )
+    parts: Final = chain.from_iterable(entry.parts for entry in entries if isinstance(entry, _MessageEntry))
+    indexed: Final = (entry for entry in entries if isinstance(entry, _IndexedEntry))
+    return sum(parts) == len(message.content or "") and all(entry.index < sizes[entry.type] for entry in indexed)
+
+
+def _resolve(
+    message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry], replay_tool_search: bool
+) -> ReplayedTurn | None:
+    indexed: Final = tuple(entry for entry in entries if isinstance(entry, _IndexedEntry))
+    if not replay_tool_search and any(entry.type in ("tool_search_call", "tool_search_output") for entry in indexed):
+        return None
+    messages: Final = tuple(entry for entry in entries if isinstance(entry, _MessageEntry))
+    calls: Final = tuple(entry for entry in indexed if entry.type in ("function_call", "custom_tool_call"))
+    texts: Final = _message_texts(messages, message.content or "")
+    positions: Final = tuple(position for position, entry in enumerate(entries) if isinstance(entry, _MessageEntry))
+    resolved: Final = _Resolved(
+        message=message,
+        texts=MappingProxyType(dict(zip(positions, texts, strict=True))),
+        tool_searches=_tool_search_entries(message),
+    )
+    tool_calls: Final = message.tool_calls or ()
+    return ReplayedTurn(
+        items=tuple(_input_item(resolved, position, entry) for position, entry in enumerate(entries)),
+        custom_call_ids=frozenset(tool_calls[entry.index].id for entry in calls if entry.type == "custom_tool_call"),
+    )
+
+
+def replay_items(message: object, *, replay_tool_search: bool) -> ReplayedTurn | None:
+    chat: Final = _chat_message(message)
+    stored: Final = _stored_replay(chat) if chat is not None and chat.role == "assistant" else None
+    if chat is None or stored is None or stored.checksum != _checksum(chat, stored.items):
+        return None
+    if not _fits(chat, stored.items):
+        return None
+    return _resolve(chat, stored.items, replay_tool_search)

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -24,9 +24,19 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 import litellm
 from litellm import ModelResponse
 from litellm._logging import verbose_logger
+from litellm.completion_extras.litellm_responses_transformation.history import (
+    PHASE_FIELD,
+    UnknownPhase,
+    output_message_text,
+    reasoning_input_item,
+    replay_fields,
+    replay_items,
+    requested_phase,
+)
 from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
 from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    prompt_cache_breakpoint_for_wire,
     responses_reasoning_items_from_thinking_blocks,
     with_prompt_cache_breakpoint,
 )
@@ -52,7 +62,6 @@ from litellm.types.llms.openai import (
     ChatCompletionToolCallChunk,
     ChatCompletionToolCallFunctionChunk,
     ChatCompletionToolParamFunctionChunk,
-    PromptCacheBreakpoint,
     Reasoning,
     ResponsesAPIOptionalRequestParams,
     ResponsesAPIResponse,
@@ -490,7 +499,7 @@ def _fold_choices_into_one(
     choices: Sequence["Choices"],
     tool_calls: Sequence[Mapping[str, object]],
     trailing_reasoning_content: str | None,
-    trailing_reasoning_item: _BuiltReasoningItem | None,
+    trailing_reasoning_items: Sequence[_BuiltReasoningItem],
     tool_search_items: Sequence[Mapping[str, object]],
     structured_output: bool = False,
 ) -> "Choices":
@@ -509,7 +518,7 @@ def _fold_choices_into_one(
     reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
     reasoning_items: Final = (
         *chain.from_iterable(_message_reasoning_items(m) for m in messages),
-        *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
+        *trailing_reasoning_items,
     )
     return Choices(
         message=Message(
@@ -526,6 +535,61 @@ def _fold_choices_into_one(
     )
 
 
+def _with_replay_fields(
+    choice: "Choices", output_items: Sequence[object], *, completed: bool, structured_output: bool
+) -> "Choices":
+    fields: Final = {
+        **(choice.message.provider_specific_fields or {}),
+        **replay_fields(output_items, choice.message, completed=completed, first_message_only=structured_output),
+    }
+    message: Final = choice.message.model_copy(update={"provider_specific_fields": fields or None})
+    return choice.model_copy(update={"message": message})
+
+
+_STREAMED_OUTPUT_ITEMS: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
+    tuple[Mapping[str, object], ...]
+)
+
+
+def completed_stream_history(
+    output_payload: object,
+    tool_search_fields: Mapping[str, object] | None,
+    *,
+    completed: bool,
+    structured_output: bool,
+) -> dict[str, object] | None:  # mutable-ok: Delta.provider_specific_fields is typed as a dict
+    items: Final = _STREAMED_OUTPUT_ITEMS.validate_python(output_payload)
+    calls: Final = tuple(item for item in items if item.get("type") in ("function_call", "custom_tool_call"))
+    streamed_message: Final = {
+        "role": "assistant",
+        "content": output_message_text(items, first_message_only=structured_output),
+        "tool_calls": [tool_call_dict_from_output_item(call, index) for index, call in enumerate(calls)],
+        "reasoning_items": list(_reasoning_items_from_output_items(items)),
+        "provider_specific_fields": tool_search_fields,
+    }
+    fields: Final = {
+        **(tool_search_fields or {}),
+        **replay_fields(items, streamed_message, completed=completed, first_message_only=structured_output),
+    }
+    return fields or None
+
+
+def _assistant_phase_field(msg: object, replay_openai_history: bool, drop_params: bool) -> Mapping[str, object]:
+    match requested_phase(msg) if replay_openai_history else None:
+        case None:
+            return {}
+        case UnknownPhase(value=value) if drop_params:
+            verbose_logger.debug("Chat provider: dropping assistant phase=%r under drop_params", value)
+            return {}
+        case UnknownPhase(value=value):
+            raise litellm.UnsupportedParamsError(
+                message=f"phase must be 'commentary' or 'final_answer', got {value!r}. "
+                "To drop it, set `litellm.drop_params=True` or pass `drop_params=True`."
+            )
+        case phase:
+            return {PHASE_FIELD: phase}
+
+
 def _responses_event_type(chunk: Mapping[str, object]) -> object:
     event_type: Final = chunk.get("type")
     return event_type.value if isinstance(event_type, ResponsesAPIStreamEvents) else event_type
@@ -567,19 +631,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li
     return "length"
 
 
-_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
-
-
-def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
-    if marker is None or not drop_params:
-        return marker
-    try:
-        return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
-    except ValidationError:
-        verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
-        return None
-
-
 def _input_file_from_file_value(file_value: object) -> dict[str, object]:
     if not isinstance(file_value, dict):
         return {"type": "input_file"}
@@ -667,15 +718,7 @@ def _reasoning_item_to_response_input(
     An item without an id is sent without one: the Responses API accepts that and
     verifies the encrypted content on its own, while it rejects any id it did not mint.
     """
-    item_id: Final = r_item.get("id")
-    encrypted_content: Final = r_item.get("encrypted_content")
-    return {
-        "type": "reasoning",
-        **({"id": item_id} if item_id else {}),
-        # summary is always required by the Responses API, even when empty
-        "summary": r_item.get("summary") or [],
-        **({"encrypted_content": encrypted_content} if encrypted_content else {}),
-    }
+    return dict(reasoning_input_item(r_item.get("id"), r_item.get("summary"), r_item.get("encrypted_content")))
 
 
 def _requested_defer_loading(tool: Mapping[str, object], function_tool: Mapping[str, object]) -> object:
@@ -804,12 +847,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         keep_prompt_cache_breakpoints: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
+        replay_openai_history: bool = False,
     ) -> tuple[list[object], str | None]:
         converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
             messages,
             drop_params=drop_params,
             replay_tool_search=replay_tool_search,
             include_namespace=include_namespace,
+            replay_openai_history=replay_openai_history,
         )
         return (
             converted_input_items
@@ -825,10 +870,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         drop_params: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
+        replay_openai_history: bool = False,
     ) -> tuple[list[object], str | None]:
         input_items: Final[list[object]] = []
         instructions: str | None = None
-        custom_tool_call_ids: Final = frozenset(
+        replayed_turns: Final = tuple(
+            replay_items(msg, replay_tool_search=replay_tool_search) if replay_openai_history else None
+            for msg in messages
+        )
+        replayed_custom_call_ids: Final = frozenset(
+            chain.from_iterable(turn.custom_call_ids for turn in replayed_turns if turn is not None)
+        )
+        custom_tool_call_ids: Final = replayed_custom_call_ids | frozenset(
             tool_call["id"]
             for msg in messages
             if msg.get("role") == "assistant" and isinstance(msg.get("tool_calls"), list)
@@ -849,7 +902,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             tool_calls = msg.get("tool_calls")
             tool_call_id = msg.get("tool_call_id")
 
-            if role == "system":
+            if (replayed_turn := replayed_turns[index]) is not None:
+                input_items.extend(replayed_turn.items)
+            elif role == "system":
                 # Extract system message as instructions
                 if isinstance(content, str) and index < leading_system_count:
                     if instructions:
@@ -915,6 +970,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             "content": self._convert_content_to_responses_format(
                                 content, "assistant", drop_params=drop_params
                             ),
+                            **_assistant_phase_field(msg, replay_openai_history, drop_params),
                         }
                     )
                 for tool_call in tool_calls:
@@ -955,6 +1011,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         "content": self._convert_content_to_responses_format(
                             content, cast(str, role), drop_params=drop_params
                         ),
+                        **_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
                     }
                 )
             elif role == "assistant":
@@ -1081,6 +1138,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
             replay_tool_search=_requests_tool_search(cast(object, optional_params)),  # cast-ok: upcast
             include_namespace=True,
+            replay_openai_history=provider == "openai",
         )
         # OpenAI's Responses API rejects an empty input. For a system-only
         # request, carry the system message as a system-role input item instead
@@ -1165,6 +1223,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         output_items: Sequence[object],
         handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None,
         structured_output: bool = False,
+        replay_openai_history: bool = False,
+        completed: bool = False,
     ) -> list["Choices"]:
         """
         Convert Responses API output items to chat completion choices.
@@ -1174,6 +1234,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             handle_raw_dict_callback: Optional callback for handling raw dict items
             structured_output: The chat request carried a json response_format, so only the
                 first message item becomes content
+            replay_openai_history: Record the output's item sequence for the next turn
+            completed: The response finished, so its item sequence is whole
 
         Returns:
             List of Choices objects
@@ -1196,7 +1258,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         choices: Final[list[Choices]] = []
         index = 0
         reasoning_content: str | None = None
-        pending_reasoning_item: _BuiltReasoningItem | None = None
+        pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = ()
 
         # Collect all tool calls to put them in a single choice
         # (Chat Completions API expects all tool calls in one message)
@@ -1205,12 +1267,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         for item in output_items:
             if isinstance(item, ResponseReasoningItem):
-                pending_reasoning_item = _build_reasoning_item(
-                    item_id=item.id,
-                    encrypted_content=getattr(item, "encrypted_content", None),
-                    summary_raw=item.summary,
+                pending_reasoning_items = (
+                    *(pending_reasoning_items if replay_openai_history else ()),
+                    _build_reasoning_item(
+                        item_id=item.id,
+                        encrypted_content=getattr(item, "encrypted_content", None),
+                        summary_raw=item.summary,
+                    ),
+                )
+                reasoning_content = " ".join(
+                    s["text"]
+                    for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items)
+                    if s.get("text")
                 )
-                reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text"))
 
             elif isinstance(item, ResponseOutputMessage):
                 for content in item.content:
@@ -1227,7 +1296,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         annotations=annotations,
                         reasoning_items=cast(
                             list[ChatCompletionReasoningItem] | None,
-                            ([pending_reasoning_item] if pending_reasoning_item is not None else None),
+                            (list(pending_reasoning_items) or None),
                         ),
                     )
 
@@ -1240,7 +1309,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     )
 
                     reasoning_content = None  # flush
-                    pending_reasoning_item = None  # flush
+                    pending_reasoning_items = ()  # flush
                     index += 1
 
             elif isinstance(item, ResponseFunctionToolCall):
@@ -1300,16 +1369,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     output_items, "stop", tool_search_items=tool_search_items
                 )
             ]
-        return [
-            _fold_choices_into_one(
-                choices=choices,
-                tool_calls=accumulated_tool_calls,
-                trailing_reasoning_content=reasoning_content,
-                trailing_reasoning_item=pending_reasoning_item,
-                tool_search_items=tool_search_items,
-                structured_output=structured_output,
-            )
-        ]
+        folded: Final = _fold_choices_into_one(
+            choices=choices,
+            tool_calls=accumulated_tool_calls,
+            trailing_reasoning_content=reasoning_content,
+            trailing_reasoning_items=pending_reasoning_items,
+            tool_search_items=tool_search_items,
+            structured_output=structured_output,
+        )
+        if not replay_openai_history:
+            return [folded]
+        return [_with_replay_fields(folded, output_items, completed=completed, structured_output=structured_output)]
 
     @staticmethod
     def _build_empty_choice(
@@ -1443,6 +1513,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             structured_output=is_structured_output_request(
                 cast("Mapping[str, object]", optional_params)  # cast-ok: optional_params is an untyped dict
             ),
+            replay_openai_history=_served_provider(
+                cast("Mapping[str, object]", litellm_params)  # cast-ok: litellm_params is an untyped dict
+            )
+            == "openai",
+            completed=raw_response.status == "completed",
         )
 
         response_is_incomplete: Final = raw_response.status == "incomplete" or (
@@ -1506,9 +1581,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         sync_stream: bool,
         json_mode: bool | None = False,
         structured_output: bool = False,
+        replays_openai_history: bool = False,
     ) -> BaseModelResponseIterator:
         return OpenAiResponsesToChatCompletionStreamIterator(
-            streaming_response, sync_stream, json_mode, structured_output=structured_output
+            streaming_response,
+            sync_stream,
+            json_mode,
+            structured_output=structured_output,
+            replays_openai_history=replays_openai_history,
         )
 
     def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
@@ -1586,7 +1666,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     if original_type == "text":
                         converted = with_prompt_cache_breakpoint(
                             self._convert_content_str_to_input_text(item.get("text", ""), role),
-                            _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   text -> %s", converted)
@@ -1599,14 +1679,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                     cast(ChatCompletionImageObject, item), role
                                 ),
                             ),
-                            _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   image_url -> %s", converted)
                     elif original_type == "input_audio":
                         converted = with_prompt_cache_breakpoint(
                             {"type": "input_audio", "input_audio": item.get("input_audio")},
-                            _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   input_audio -> %s", converted)
@@ -1622,7 +1702,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                 _input_file_from_file_value(
                                     cast("ChatCompletionFileObject", item).get("file"),  # cast-ok: type tag checked
                                 ),
-                                _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   file -> %s", converted)
@@ -1646,7 +1726,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             # Default to input_text for unknown types
                             converted = with_prompt_cache_breakpoint(
                                 self._convert_content_str_to_input_text(str(item.get("text", item)), role),
-                                _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   unknown(%s) -> %s", original_type, converted)
@@ -1890,8 +1970,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         sync_stream: bool,
         json_mode: bool | None = False,
         structured_output: bool = False,
+        replays_openai_history: bool = False,
     ):
         super().__init__(streaming_response, sync_stream, json_mode)
+        self._replays_openai_history: Final = replays_openai_history
         self._chat_completion_id: str | None = None
         self._served_service_tier: str | None = None
         self._tool_call_index_map: dict[int, int] = {}  # mutable-ok: per-stream accumulator state
@@ -1937,6 +2019,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
     def translate_responses_chunk_to_openai_stream(
         parsed_chunk: dict | BaseModel,
         tool_call_index_map: dict[int, int] | None = None,  # mutable-ok: per-stream state, remapped in place
+        replays_openai_history: bool = False,
+        structured_output: bool = False,
     ) -> "ModelResponseStream":
         """
         Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
@@ -1944,6 +2028,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         Args:
             parsed_chunk: Dict containing the Responses API event chunk
             tool_call_index_map: Per-stream output_index -> sequential tool_call index map
+            replays_openai_history: Record the completed output's item sequence for the next turn
+            structured_output: Only the first message item became content
 
         Returns:
             ModelResponseStream: OpenAI-formatted streaming chunk
@@ -2154,6 +2240,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             terminal_reasoning_items_typed: Final = _as_chat_reasoning_items(
                 _reasoning_items_from_output_items(output_items)
             )
+            output_payload: Final = cast(object, output_items)  # cast-ok: untyped
+            tool_search_fields: Final = _tool_search_provider_fields(
+                _tool_search_items_from_output_items(output_payload)
+            )
 
             usage = None
             if response_data.get("usage"):
@@ -2167,10 +2257,17 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                     StreamingChoices(
                         index=0,
                         delta=Delta(
-                            content="",
+                            content=None if replays_openai_history else "",
                             reasoning_items=terminal_reasoning_items_typed,
-                            provider_specific_fields=_tool_search_provider_fields(
-                                _tool_search_items_from_output_items(cast(object, output_items))  # cast-ok: untyped
+                            provider_specific_fields=(
+                                completed_stream_history(
+                                    output_payload,
+                                    tool_search_fields,
+                                    completed=event_type == "response.completed",
+                                    structured_output=structured_output,
+                                )
+                                if replays_openai_history
+                                else tool_search_fields
                             ),
                         ),
                         finish_reason=finish_reason,
@@ -2216,7 +2313,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             self._empty_content_chunk()
             if self._is_text_delta_after_structured_answer(chunk)
             else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
-                chunk, tool_call_index_map=self._tool_call_index_map
+                chunk,
+                tool_call_index_map=self._tool_call_index_map,
+                replays_openai_history=self._replays_openai_history,
+                structured_output=self._structured_output,
             )
         )
         if self._structured_output and _is_message_item_done(chunk):

litellm/constants.py

@@ -158,7 +158,7 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_
 DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250)
 HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
 HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range(
-    "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400
+    "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 60, 1, 86400
 )
 
 # Maximum wall-clock seconds a streaming response is allowed to run.

litellm/integrations/anthropic_cache_control_hook.py

@@ -72,6 +72,8 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
 OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
 _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
+_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
+_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
 OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
     {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"}
 )
@@ -125,6 +127,10 @@ def _served_provider_for_model(model: str) -> str | None:
 
 
 def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
+    return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True
+
+
+def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None:
     hosted_flag: Final = (
         None
         if custom_llm_provider is None
@@ -135,9 +141,19 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str
     model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
     if model_map_flag is not None:
         return model_map_flag
-    version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower())
-    if version_match is None:
+    fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1])
+    if fine_tuned_base is not None:
+        return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None
+    return _named_model_breakpoint_support(model)
+
+
+def _named_model_breakpoint_support(model: str) -> bool | None:
+    bare_model: Final = model.rsplit("/", 1)[-1].lower()
+    if _O_SERIES_PATTERN.match(bare_model) is not None:
         return False
+    version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
+    if version_match is None:
+        return None
     version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
     return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
 

litellm/integrations/prometheus.py

@@ -1173,8 +1173,8 @@ class PrometheusLogger(CustomLogger):
         # Build label filters from valid configurations
         label_filters: Final = self._build_label_filters(parsed_configs)
 
-        # Pretty print the processed configuration
-        self._pretty_print_prometheus_config(label_filters)
+        if litellm.suppress_startup_output is not True:
+            self._pretty_print_prometheus_config(label_filters)
         return label_filters
 
     def _parse_exclude_config(self) -> tuple[frozenset[str], frozenset[str]]:

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
 )
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
 from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
+from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
 from litellm.types.utils import LlmProviders
 
 from ..exceptions import (
@@ -2762,6 +2763,8 @@ def exception_type(
                     exception_provider=exception_provider,
                     extra_information=extra_information,
                 )
+            if custom_llm_provider == "bigmodel":
+                raise_bigmodel_business_error(model, f"{exception_provider} - {error_str}", original_exception)  # pyright: ignore[reportUnknownArgumentType]  # exception_type takes untyped parameters
             if custom_llm_provider == "openrouter":
                 _map_openrouter_exception(
                     model=model,

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -794,6 +794,13 @@ def _get_openai_compatible_provider_info(
             api_base,
             dynamic_api_key,
         ) = litellm.XAIChatConfig().get_openai_compatible_provider_info(api_base, api_key)
+    elif custom_llm_provider == "bigmodel":
+        from litellm.llms.bigmodel.common_utils import get_bigmodel_api_key, get_bigmodel_url
+
+        (
+            api_base,  # rebind-ok: existing provider-resolution pipeline resolves the API base in place
+            dynamic_api_key,  # rebind-ok: existing provider-resolution pipeline resolves the API key in place
+        ) = get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
     elif custom_llm_provider == "zai":
         (
             api_base,

litellm/litellm_core_utils/internal_call_metadata.py

@@ -120,7 +120,7 @@ def is_unbilled_non_inference_call(
     origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) if metadata is not None else None
     if origin == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN:
         return False
-    if not is_background_response(response):
+    if not is_background_response(_finished_response(response)):
         return True
     return origin == BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN or _is_managed_background_call(metadata)
 

litellm/litellm_core_utils/llm_request_utils.py

@@ -1,8 +1,9 @@
-from collections.abc import Mapping
+from collections.abc import Iterable, Iterator, Mapping
 from typing import Final, cast
 
 import litellm
 from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
+from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
 
 
 def _form_field_value(value: object) -> str:
@@ -153,24 +154,33 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
     if custom_llm_provider not in litellm.models_by_provider:
         return []
 
-    known_models: Final = litellm.models_by_provider.get(custom_llm_provider, [])
-    model_costs: Final = []
+    known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
+    ranked_rows: Final = (
+        (route_rank, cost, model)
+        for model, cost in _chat_row_costs(known_models, custom_llm_provider)
+        if (route_rank := _route_rank(model, custom_llm_provider)) is not None
+    )
+    return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
 
-    for model in known_models:
+
+def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
+    for model in models:
         try:
             model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
         except Exception:
             continue
         if model_info.get("mode") != "chat":
             continue
-        _cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
-        model_costs.append((model, _cost))
+        yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
 
-    # Sort by cost (ascending)
-    model_costs.sort(key=lambda x: x[1])
 
-    # Return the top n cheapest models
-    return [model for model, _ in model_costs[:n]]
+def _route_rank(model: str, custom_llm_provider: str) -> int | None:
+    routed_provider: Final = inferred_provider(model)
+    if routed_provider is None:
+        return 1
+    if routed_provider == inferred_provider(f"{custom_llm_provider}/{model}"):
+        return 0
+    return None
 
 
 def get_proxy_server_request_headers(litellm_params: dict | None) -> dict:

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -17,7 +17,7 @@ from pathlib import Path
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
 
-from pydantic import TypeAdapter
+from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm import verbose_logger
@@ -2073,6 +2073,19 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
     return cast(_MarkedT, marked)  # cast-ok: same block shape as the input plus the marker key
 
 
+_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
+
+
+def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
+    if marker is None or not drop_params:
+        return marker
+    try:
+        return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
+    except ValidationError:
+        verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
+        return None
+
+
 def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
     """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block."""
     if block.get("cache_control") is None:

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -22,6 +22,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRAC
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_content_from_model_response,  # pyright: ignore[reportUnknownVariableType]  # response param accepts an untyped dict
 )
+from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionAssistantContentValue,
@@ -55,7 +56,7 @@ from litellm.types.utils import (
     UnreportedUsage,
     Usage,
 )
-from litellm.utils import print_verbose, token_counter
+from litellm.utils import get_base_model_from_metadata, print_verbose, token_counter
 
 if TYPE_CHECKING:
     from openai.types.completion_usage import CompletionUsage
@@ -1556,6 +1557,9 @@ def stream_chunk_builder(
             return stream_chunk_builder_text_completion(chunks=chunks, messages=messages)
 
         model: Final = chunks[0]["model"]
+        usage_model: Final = usage_tokenizer_model(
+            model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
+        )
         # Initialize the response dictionary
         response: Final = processor.build_base_response(chunks)
 
@@ -1579,7 +1583,7 @@ def stream_chunk_builder(
             completion_output = get_content_from_model_response(response)
             usage = processor.calculate_usage(
                 chunks=chunks,
-                model=model,
+                model=usage_model,
                 completion_output=completion_output,
                 messages=messages,
                 reasoning_tokens=0,
@@ -1770,7 +1774,7 @@ def stream_chunk_builder(
 
         usage = processor.calculate_usage(
             chunks=chunks,
-            model=model,
+            model=usage_model,
             completion_output=completion_output,
             messages=messages,
             reasoning_tokens=reasoning_tokens,

litellm/litellm_core_utils/streaming_handler.py

@@ -1938,6 +1938,16 @@ class CustomStreamWrapper:
         model_response: Final = self.model_response_creator(
             hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
         )
+        served_tier: Final = next(
+            (
+                tier
+                for chunk in reversed(self.chunks)
+                if isinstance(tier := getattr(chunk, "service_tier", None), str) and tier
+            ),
+            None,
+        )
+        if served_tier is not None:
+            setattr(model_response, "service_tier", served_tier)  # noqa: B010  # pydantic extra, not a declared field
         _finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason
         if _finish_reason is not None:
             model_response.choices[0].finish_reason = _finish_reason

litellm/litellm_core_utils/token_counter.py

@@ -15,6 +15,7 @@ import anyio
 import anyio.lowlevel
 import httpx
 import tiktoken
+from tiktoken.model import MODEL_TO_ENCODING
 from typing_extensions import ParamSpec, TypeVar
 
 import litellm
@@ -49,6 +50,7 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace
 from litellm.litellm_core_utils.url_utils import safe_get
 from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value
+from litellm.router_utils.deployment_pricing import deployment_pricing_models
 from litellm.rust_bridge.tokenizer import get_encoding
 from litellm.types.llms.anthropic import (
     AnthropicContentParamSource,
@@ -609,7 +611,7 @@ T_ParamSpec: Final = ParamSpec("T_ParamSpec")
 T_Retval = TypeVar("T_Retval")
 _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter")
 _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset(
-    ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming")
+    ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming", "custom")
 )
 
 
@@ -1005,17 +1007,144 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
 
 
 def openai_tokenizer_encoding_name(model: str) -> str:
-    """The tiktoken encoding name for `model`, without loading the encoding."""
-    from litellm.utils import print_verbose
+    """The tiktoken encoding name for `model`, without loading the encoding.
 
-    model_to_use: Final = _fix_model_name(model)
+    A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does
+    know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every
+    OpenAI model since gpt-4o uses. Provider and region prefixes (`azure/eu/`, `azure_ai/`) are dropped, a name
+    listed only under Azure (`gpt-41`, `openai/o1-mini`) counts as listed through its `azure/` cost-map key, and an
+    unlisted name tiktoken counts with o200k_base (`openai/gpt-5.9`) or named like a gpt-5 or o-series release
+    (`openai/gpt-6.2`) gets it."""
+    model_to_use: Final = _openai_model_name(model) or _fix_model_name(model)
     if "gpt-4o" in model_to_use:
         return "o200k_base"
     try:
         return tiktoken.encoding_name_for_model(model_to_use)
     except KeyError:
-        print_verbose("Warning: model not found. Using cl100k_base encoding.")
-        return "cl100k_base"
+        return _suffixed_variant_encoding_name(model_to_use) or "o200k_base"
+
+
+def unresolved_tokenizer_model(model: str) -> str:
+    """`model`, or `gpt-3.5-turbo` when resolving the OpenAI model `model` names moves it off cl100k_base, the encoding
+    tiktoken gives the name as written, so a count with the result never drops below that one."""
+    as_written: Final = _fix_model_name(model)
+    as_written_encoding: Final = (
+        "o200k_base" if "gpt-4o" in as_written else (_tiktoken_encoding_name(as_written) or "cl100k_base")
+    )
+    moved_off_cl100k: Final = (
+        as_written_encoding == "cl100k_base" and openai_tokenizer_encoding_name(model) != "cl100k_base"
+    )
+    return "gpt-3.5-turbo" if moved_off_cl100k else model
+
+
+def usage_tokenizer_model(model: str, base_model: str | None) -> str:
+    """The model to estimate missing usage with: a deployment's `base_model` when it names an OpenAI model, since a
+    custom deployment name says nothing about its tokenizer, else `model`."""
+    return base_model if base_model is not None and is_openai_model(base_model) else model
+
+
+def is_openai_model(model: str) -> bool:
+    """Whether `model` names an OpenAI or Azure OpenAI model, listed in the cost map or counted with o200k_base by
+    tiktoken, with or without its prefixes."""
+    return _openai_model_name(model) is not None
+
+
+def deployment_openai_model(deployment: object) -> str | None:
+    """The OpenAI model a router deployment serves, read from its base model first, since a custom deployment name
+    (`azure/my-gpt-5`) says nothing about its tokenizer, or None when it serves no OpenAI model."""
+    return _as_openai_model(_served_model(deployment))
+
+
+def served_openai_models(deployments: Iterable[object]) -> tuple[str | None, ...]:
+    """The distinct OpenAI models `deployments` serve, with None standing for any deployment serving another model."""
+    served: Final = dict.fromkeys(_served_model(deployment) for deployment in deployments)
+    return tuple(dict.fromkeys(_as_openai_model(model) for model in served))
+
+
+def _served_model(deployment: object) -> str | None:
+    return next(iter(deployment_pricing_models(deployment)), None)
+
+
+def _as_openai_model(model: str | None) -> str | None:
+    return model if model is not None and is_openai_model(model) else None
+
+
+def single_encoding_openai_model(served: Sequence[str | None]) -> str | None:
+    """An OpenAI model to count a request with, when every `served_openai_models` entry is an OpenAI model on the same
+    encoding, else None."""
+    if not served or None in served:
+        return None
+    encodings: Final = frozenset(openai_tokenizer_encoding_name(model) for model in served if model is not None)
+    return served[0] if len(encodings) == 1 else None
+
+
+def _openai_model_name(model: str) -> str | None:
+    return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model)
+
+
+_VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)")
+
+
+def _versioned_openai_model_name(model: str) -> str | None:
+    name: Final = model.rsplit("/", 1)[-1]
+    return name if _VERSIONED_OPENAI_MODEL.match(name) else None
+
+
+def _tiktoken_o200k_model_name(model: str) -> str | None:
+    name: Final = model.rsplit("/", 1)[-1]
+    return name if _tiktoken_encoding_name(name) == "o200k_base" else None
+
+
+def _listed_openai_model_name(model: str) -> str | None:
+    segments: Final = model.split("/")
+    suffixes: Final = ("/".join(segments[start:]) for start in range(len(segments)))
+    listed: Final = any(
+        _is_openai_cost_map_model(suffix) or _is_openai_cost_map_model(f"azure/{suffix}") for suffix in suffixes
+    )
+    return segments[-1] if listed else None
+
+
+def _is_openai_cost_map_model(model: str) -> bool:
+    return (
+        model in litellm.azure_llms
+        or model in litellm.open_ai_chat_completion_models
+        or _is_azure_openai_text_model(model)
+    )
+
+
+_TEXT_MODEL_MODES: Final = frozenset({"chat", "completion", "responses", "realtime"})
+
+
+def _is_azure_openai_text_model(model: str) -> bool:
+    cost_info: Final[Mapping[str, object]] = litellm.model_cost.get(model) or {}
+    return (
+        cost_info.get("litellm_provider") == "azure"
+        and cost_info.get("mode") in _TEXT_MODEL_MODES
+        and _is_openai_model_name(model.rsplit("/", 1)[-1])
+    )
+
+
+def _is_openai_model_name(name: str) -> bool:
+    return (
+        name in litellm.open_ai_chat_completion_models
+        or _tiktoken_encoding_name(name) is not None
+        or any(
+            name.startswith(f"{listed}-") or listed.startswith(f"{name}-")
+            for listed in tuple(litellm.open_ai_chat_completion_models)
+        )
+    )
+
+
+def _tiktoken_encoding_name(model: str) -> str | None:
+    try:
+        return tiktoken.encoding_name_for_model(model)
+    except KeyError:
+        return None
+
+
+def _suffixed_variant_encoding_name(model: str) -> str | None:
+    known: Final = tuple(name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-"))
+    return MODEL_TO_ENCODING[max(known, key=len)] if known else None
 
 
 def uses_legacy_message_accounting(model: str) -> bool:

litellm/llms/anthropic/common_utils.py

@@ -121,6 +121,54 @@ _DECODED_JSON: Final = TypeAdapter(object)
 _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
 
 
+def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
+    if custom_llm_provider == "anthropic":
+        return True
+    model_lower: Final = model.lower()
+    if custom_llm_provider == "bedrock":
+        return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower)
+    if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai":
+        return "claude" in model_lower
+    return False
+
+
+def is_claude_model_target(custom_llm_provider: str | None, model: str) -> bool:
+    return "claude" in model.lower() and is_claude_tool_target(custom_llm_provider, model)
+
+
+def _hoistable_defer_loading(tool_dict: Mapping[object, object], custom: object) -> bool | None:
+    if not isinstance(custom, dict) or "defer_loading" in tool_dict:
+        return None
+    defer_loading: Final = OBJECT_MAPPING_ADAPTER.validate_python(custom).get("defer_loading")
+    return defer_loading if isinstance(defer_loading, bool) else None
+
+
+def _tool_without_custom_field(tool: object, hoist_defer_loading: bool, drop_custom: bool) -> object:
+    if not isinstance(tool, dict):
+        return tool
+
+    if "custom" not in tool:
+        return tool
+
+    tool_dict: Final = OBJECT_MAPPING_ADAPTER.validate_python(tool)
+
+    custom: Final = tool_dict["custom"]
+    without_custom: Final = {key: value for key, value in tool_dict.items() if key != "custom"}
+    defer_loading: Final = _hoistable_defer_loading(tool_dict, custom) if hoist_defer_loading else None
+    if defer_loading is None:
+        return without_custom if drop_custom else tool_dict
+
+    unhoisted: Final = {
+        key: value for key, value in OBJECT_MAPPING_ADAPTER.validate_python(custom).items() if key != "defer_loading"
+    }
+    kept_custom: Final = {"custom": unhoisted} if unhoisted and not drop_custom else {}
+    return {**without_custom, "defer_loading": defer_loading, **kept_custom}
+
+
+def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: bool, drop_custom: bool) -> list[object]:
+    return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
+
+
 def is_anthropic_messages_url(url: str) -> bool:
     """Check whether a URL addresses Anthropic's Messages API."""
     parsed_url: Final = urlparse(url)

litellm/llms/anthropic/count_tokens/handler.py

@@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
         tools: list[dict[str, JsonValue]] | None = None,
         system: JsonValue = None,
         optional_params: Mapping[str, JsonValue] | None = None,
+        drop_params: bool = False,
     ) -> dict[str, JsonValue]:
         """
         Handle a CountTokens request using httpx.
@@ -69,6 +70,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
                 tools=tools,
                 system=system,
                 optional_params=optional_params,
+                drop_params=drop_params,
             )
 
             verbose_logger.debug("Transformed request: %s", request_body)

litellm/llms/anthropic/count_tokens/token_counter.py

@@ -11,6 +11,7 @@ from pydantic import JsonValue
 import litellm
 from litellm._logging import verbose_logger
 from litellm.exceptions import AuthenticationError
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
 from litellm.llms.anthropic.count_tokens.transformation import (
     AnthropicCountFields,
@@ -131,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
                 tools=tools,
                 system=system,
                 optional_params=count_options,
+                drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
             )
 
             if result is not None:

litellm/llms/anthropic/count_tokens/transformation.py

@@ -26,7 +26,11 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     split_leading_system_run,
 )
 from litellm.litellm_core_utils.url_utils import join_api_base
-from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers
+from litellm.llms.anthropic.common_utils import (
+    is_claude_model_target,
+    merge_anthropic_beta_headers,
+    tools_without_custom_field,
+)
 from litellm.llms.anthropic.wif import resolve_anthropic_base
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes
 from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
@@ -372,6 +376,7 @@ class AnthropicCountTokensConfig:
         tools: list[dict[str, JsonValue]] | None = None,
         system: JsonValue = None,
         optional_params: Mapping[str, JsonValue] | None = None,
+        drop_params: bool = False,
     ) -> dict[str, JsonValue]:  # mutable-ok: provider transport requires JSON dictionaries
         """
         Transform request to Anthropic CountTokens format.
@@ -391,7 +396,19 @@ class AnthropicCountTokensConfig:
                     **MappingProxyType(
                         {
                             key: value
-                            for key, value in (("system", counted_system), ("tools", tools))
+                            for key, value in (
+                                ("system", counted_system),
+                                (
+                                    "tools",
+                                    None
+                                    if tools is None
+                                    else tools_without_custom_field(
+                                        tools,
+                                        hoist_defer_loading=is_claude_model_target("anthropic", model),
+                                        drop_custom=drop_params,
+                                    ),
+                                ),
+                            )
                             if value is not None
                         }
                     ),

litellm/llms/anthropic/messages/adapters/transformation.py

@@ -820,6 +820,7 @@ class LiteLLMAnthropicMessagesAdapter:
             "name",
             "input_schema",
             "description",
+            "custom",
             "cache_control",
             "strict",
             "type",

litellm/llms/anthropic/messages/native/transformation.py

@@ -29,13 +29,16 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
 from litellm.types.router import GenericLiteLLMParams
 
 from ...common_utils import (
+    OBJECT_LIST_ADAPTER,
     AnthropicError,
     AnthropicModelInfo,
+    is_claude_model_target,
     merge_anthropic_beta_headers,
     optionally_handle_anthropic_oauth,
     requires_native_compaction_beta,
     strip_advisor_blocks_from_messages,
     strip_encrypted_reasoning_blocks_from_anthropic_messages,
+    tools_without_custom_field,
 )
 from .mid_conversation_system import (
     as_system_content_blocks,
@@ -181,6 +184,16 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         """
         return False
 
+    def hoists_wrapped_defer_loading(self, model: str) -> bool:
+        return is_claude_model_target(self._resolved_provider, model)
+
+    def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]:
+        return tools_without_custom_field(
+            tools,
+            hoist_defer_loading=self.hoists_wrapped_defer_loading(model),
+            drop_custom=drops_unsupported_params(drop_params),
+        )
+
     @staticmethod
     def _filter_billing_headers_from_system(system_param):
         """
@@ -700,7 +713,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
 
         # Auto-strip advisor blocks from history if advisor tool is absent.
         # Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
-        _tools: Final = anthropic_messages_optional_request_params.get("tools") or []
+        tools: Final = anthropic_messages_optional_request_params.get("tools")
+        _tools: Final = tools or []
         _has_advisor: Final = any(isinstance(t, dict) and t.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for t in _tools)
         if not _has_advisor:
             messages = strip_advisor_blocks_from_messages(messages)
@@ -713,7 +727,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             model=model,
             **anthropic_messages_optional_request_params,
         )
-        return dict(anthropic_messages_request)
+        if not isinstance(tools, list):
+            return dict(anthropic_messages_request)
+        return {
+            **anthropic_messages_request,
+            "tools": self.tools_for_upstream(
+                OBJECT_LIST_ADAPTER.validate_python(tools), model, litellm_params.drop_params
+            ),
+        }
 
     def transform_anthropic_messages_response(
         self,

litellm/llms/anthropic/messages/responses_adapters/handler.py

@@ -15,7 +15,11 @@ from litellm.integrations.anthropic_cache_control_hook import (
     supports_openai_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
-from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    cache_control_as_prompt_cache_breakpoint,
+    prompt_cache_breakpoint_for_wire,
+    with_prompt_cache_breakpoint,
+)
 from litellm.responses.utils import auto_reasoning_summary
 from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
 from litellm.types.llms.anthropic import (
@@ -90,14 +94,28 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str
     return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
 
 
+def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]:
+    if "prompt_cache_breakpoint" not in block:
+        return cache_control_as_prompt_cache_breakpoint(block)
+    wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True)
+    return cache_control_as_prompt_cache_breakpoint(
+        with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker)
+    )
+
+
 def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None:
     """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints
     drops raw ones under drop_params."""
+    drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params"))
     if _reads_prompt_cache_breakpoint(model, forwarded_kwargs):
-        return cache_control_as_prompt_cache_breakpoint
+        return (
+            _cache_control_as_wire_safe_prompt_cache_breakpoint
+            if drops_params
+            else cache_control_as_prompt_cache_breakpoint
+        )
     if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs):
         return None
-    if drops_unsupported_params(forwarded_kwargs.get("drop_params")):
+    if drops_params:
         return _without_prompt_cache_breakpoint
     return None
 

litellm/llms/azure/azure.py

@@ -33,6 +33,7 @@ from litellm.types.utils import (
 from litellm.utils import (
     CustomStreamWrapper,
     convert_to_model_response_object,
+    get_base_model_from_metadata,
 )
 
 from ...types.llms.openai import HttpxBinaryResponseContent
@@ -145,6 +146,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         azure_client: AzureOpenAI | OpenAI,
         data: dict,
         timeout: float | httpx.Timeout,
+        logging_obj: LiteLLMLoggingObj,
     ):
         """
         Helper to:
@@ -160,7 +162,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         except BadRequestError as e:
             if not is_output_token_limit_error(e):
                 raise
-            return build_output_token_limit_response(e=e, data=data, is_async=False)
+            return build_output_token_limit_response(
+                e=e,
+                data=data,
+                is_async=False,
+                base_model=get_base_model_from_metadata(logging_obj.model_call_details),
+            )
         except Exception as e:
             raise e
 
@@ -192,7 +199,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         except BadRequestError as e:
             if not is_output_token_limit_error(e):
                 raise
-            return build_output_token_limit_response(e=e, data=data, is_async=True)
+            return build_output_token_limit_response(
+                e=e,
+                data=data,
+                is_async=True,
+                base_model=get_base_model_from_metadata(logging_obj.model_call_details),
+            )
         except Exception as e:
             raise e
 
@@ -350,7 +362,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                     )
 
                 headers, response = self.make_sync_azure_openai_chat_completion_request(
-                    azure_client=azure_client, data=data, timeout=timeout
+                    azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj
                 )
                 if not isinstance(response, BaseModel):
                     raise AzureOpenAIError(
@@ -564,7 +576,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             },
         )
         headers, response = self.make_sync_azure_openai_chat_completion_request(
-            azure_client=azure_client, data=data, timeout=timeout
+            azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj
         )
         logging_obj.model_call_details["response_headers"] = headers
         streamwrapper: Final = CustomStreamWrapper(

litellm/llms/azure/responses/transformation.py

@@ -335,7 +335,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         """
         Transform the get response API request into a URL and data
 
@@ -343,7 +343,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         - GET /v1/responses/{response_id}
         """
         get_url: Final = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id)
-        data: Final[dict] = {}
+        data: Final[dict[str, str]] = {}
         verbose_logger.debug("get response url=%s", get_url)
         return get_url, data
 

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -13,6 +13,7 @@ from pydantic import JsonValue
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.asyncify import asyncify
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.llms.anthropic.common_utils import AnthropicError
 from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
     AzureAIAnthropicCountTokensConfig,
@@ -69,6 +70,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
                 tools=tools,
                 system=system,
                 optional_params=optional_params,
+                drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")),
             )
 
             verbose_logger.debug("Transformed request: %s", request_body)

litellm/llms/base_llm/responses/transformation.py

@@ -69,6 +69,9 @@ class BaseResponsesAPIConfig(ABC):
     def supports_encrypted_agent_messages(self) -> bool:
         return False
 
+    def supports_streamed_retrieval(self) -> bool:
+        return self.custom_llm_provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI)
+
     def sign_request(
         self,
         headers: dict,
@@ -209,7 +212,7 @@ class BaseResponsesAPIConfig(ABC):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         pass
 
     @abstractmethod
@@ -268,6 +271,9 @@ class BaseResponsesAPIConfig(ABC):
         """Returns True if litellm should fake a stream for the given model and stream value"""
         return False
 
+    def may_answer_stream_with_json(self) -> bool:
+        return False
+
     def supports_native_websocket(self) -> bool:
         """
         Returns True if the provider has a native WebSocket endpoint for Responses API.

litellm/llms/bedrock/common_utils.py

@@ -15,7 +15,11 @@ from dataclasses import dataclass
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
 
-from typing_extensions import ReadOnly, TypedDict
+from typing_extensions import (
+    ReadOnly,
+    TypedDict,
+    TypeIs,  # noqa: TID251  # narrows the caller's tools without copying them
+)
 
 if TYPE_CHECKING:
     from botocore.eventstream import EventStreamMessage
@@ -573,6 +577,26 @@ def strip_unsupported_bedrock_invoke_output_config_keys(
         request_body["output_config"] = {"format": preserved_format}  # rebind-ok: out-param
 
 
+_TOOL_ITEMS_ADAPTER: Final = TypeAdapter(tuple[object, ...])
+
+
+def _is_str_keyed_dict(
+    value: object,
+) -> TypeIs[dict[str, object]]:  # guard-ok: JSON tool objects  # mutable-ok: fallbacks reuse tools edited in place
+    return isinstance(value, dict)
+
+
+def _normalize_custom_field_on_tool(tool: object) -> None:
+    if not _is_str_keyed_dict(tool):
+        return
+    custom: Final = tool.pop("custom", None)
+    if not _is_str_keyed_dict(custom) or "defer_loading" in tool:
+        return
+    deferred: Final = custom.get("defer_loading")
+    if isinstance(deferred, bool):
+        tool["defer_loading"] = deferred  # rebind-ok: a fallback reuses the caller's tool, as on main
+
+
 def normalize_custom_field_on_tools(request_body: dict) -> None:
     """
     Drop the ``custom`` field from each tool, first hoisting a boolean
@@ -590,15 +614,8 @@ def normalize_custom_field_on_tools(request_body: dict) -> None:
     tools: Final = request_body.get("tools")
     if not tools or not isinstance(tools, list):
         return
-    for tool in tools:
-        if not isinstance(tool, dict):
-            continue
-        custom: dict[str, object] | None = tool.pop("custom", None)
-        if not isinstance(custom, dict) or "defer_loading" in tool:
-            continue
-        deferred: object = custom.get("defer_loading")
-        if isinstance(deferred, bool):
-            tool["defer_loading"] = deferred
+    for tool in _TOOL_ITEMS_ADAPTER.validate_python(tools):
+        _normalize_custom_field_on_tool(tool)
 
 
 _TOOL_DICTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])

litellm/llms/bedrock/count_tokens/mantle_handler.py

@@ -8,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.anthropic.common_utils import tools_without_custom_field
 from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
 from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url
@@ -26,6 +27,7 @@ class MantleCountTokensRequest(TypedDict):
 _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest)
 _COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue])
 _COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue])
+_TOOLS: Final = TypeAdapter(list[dict[str, JsonValue]])
 
 
 class BedrockMantleCountTokensHandler(BaseAWSLLM):
@@ -59,10 +61,15 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
                 self.get_aws_region_name, optional_params=litellm_params, model=resolved_model, model_id=None
             )
             endpoint_url: Final = await run_aws_signing(self.get_mantle_count_tokens_endpoint, aws_region_name)
+            tools: Final = request.get("tools")
             body: Final = self._anthropic_config.transform_request_to_count_tokens(
                 model=resolved_model,
                 messages=request["messages"],
-                tools=request.get("tools"),
+                tools=None
+                if tools is None
+                else _TOOLS.validate_python(
+                    tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True)
+                ),
                 system=request.get("system"),
                 optional_params=_COUNT_OPTIONS.validate_python(
                     {name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES}

litellm/llms/bedrock/count_tokens/transformation.py

@@ -12,6 +12,7 @@ from typing import Final, Literal
 from pydantic import JsonValue
 
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.anthropic.common_utils import tools_without_custom_field
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.bedrock.common_utils import get_bedrock_base_model
 
@@ -194,7 +195,15 @@ class BedrockCountTokensConfig(BaseAWSLLM):
 
         # For InvokeModel, we need to provide the raw body that would be sent to the model
         # Remove the 'model' field from the body as it's not part of the model input
-        body_data: Final = {k: v for k, v in request_data.items() if k != "model"}
+        tools: Final = request_data.get("tools")
+        body_data: Final = {
+            **{k: v for k, v in request_data.items() if k != "model"},
+            **(
+                {"tools": tools_without_custom_field(tools, hoist_defer_loading=True, drop_custom=True)}
+                if isinstance(tools, list)
+                else {}
+            ),
+        }
 
         if "messages" in body_data:
             # Bedrock validates the body against the model's InvokeModel schema;

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py

@@ -102,6 +102,9 @@ class AmazonAnthropicClaudeMessagesConfig(
     def should_strip_billing_metadata(self) -> bool:
         return True
 
+    def tools_for_upstream(self, tools: Sequence[object], model: str, drop_params: object) -> Sequence[object]:
+        return tools
+
     def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
         return True
 

litellm/llms/bigmodel/chat/transformation.py

@@ -0,0 +1,59 @@
+from collections.abc import Mapping
+from typing import Final
+
+from litellm.llms.zai.chat.transformation import ZAIChatConfig
+
+from ..common_utils import get_bigmodel_api_key, get_bigmodel_url, is_unsupported_disabled_thinking
+
+_SUPPORTED_PARAMS: Final = (
+    "max_tokens",
+    "max_completion_tokens",
+    "stream",
+    "stream_options",
+    "temperature",
+    "top_p",
+    "stop",
+    "tools",
+    "tool_choice",
+    "response_format",
+    "thinking",
+    "reasoning_effort",
+)
+
+
+class BigModelChatConfig(ZAIChatConfig):
+    @property
+    def custom_llm_provider(self) -> str:
+        return "bigmodel"
+
+    def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
+        return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
+
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: inherited interface requires a list
+        return list(_SUPPORTED_PARAMS)
+
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: request pipeline adds parameters to the returned body
+        mapped: Final = {
+            **optional_params,
+            **{
+                ("max_tokens" if key == "max_completion_tokens" else key): value
+                for key, value in non_default_params.items()
+                if key in _SUPPORTED_PARAMS and key != "thinking"
+            },
+        }
+        if "thinking" not in non_default_params:
+            return mapped
+        thinking: Final = non_default_params["thinking"]
+        if is_unsupported_disabled_thinking(model, thinking):
+            return {
+                **mapped,
+                "reasoning_effort": mapped.get("reasoning_effort", "low"),
+                "extra_body": {"thinking": {"type": "enabled"}},
+            }
+        return {**mapped, "extra_body": {"thinking": thinking}}

litellm/llms/bigmodel/common_utils.py

@@ -0,0 +1,92 @@
+import re
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final, Literal
+from urllib.parse import urlsplit, urlunsplit
+
+import httpx
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+import litellm
+from litellm.exceptions import ContentPolicyViolationError, ContextWindowExceededError
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+
+BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn"
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_BUSINESS_CODE_IN_ERROR: Final = re.compile(r"""['"]code['"]\s*:\s*['"]?(\d{4})\b""")
+_CODING_CHAT_PATH: Final = "/api/coding/paas/v4"
+_API_PATHS: Final = (
+    "/api/coding/paas/v4/chat/completions",
+    _CODING_CHAT_PATH,
+    "/api/paas/v4/chat/completions",
+    "/api/anthropic/v1/messages",
+    "/api/anthropic/v1",
+    "/api/v1/responses",
+    "/api/paas/v4",
+    "/api/anthropic",
+    "/api/v1",
+    "/api",
+)
+_ENDPOINT_PATHS: Final = MappingProxyType(
+    {
+        "chat": "/api/paas/v4",
+        "responses": "/api/v1/responses",
+        "messages": "/api/anthropic/v1/messages",
+    }
+)
+
+
+def get_bigmodel_api_key(api_key: str | None) -> str | None:
+    return api_key or get_secret_str("BIGMODEL_API_KEY")
+
+
+def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses", "messages"]) -> str:
+    resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE
+    parsed: Final = urlsplit(resolved_base)
+    path: Final = parsed.path.rstrip("/")
+    suffix: Final = next((candidate for candidate in _API_PATHS if path.endswith(candidate)), "")
+    endpoint_path: Final = (
+        _CODING_CHAT_PATH if endpoint == "chat" and suffix.startswith(_CODING_CHAT_PATH) else _ENDPOINT_PATHS[endpoint]
+    )
+    return urlunsplit(
+        (parsed.scheme, parsed.netloc, f"{path.removesuffix(suffix)}{endpoint_path}", parsed.query, parsed.fragment)
+    )
+
+
+def bigmodel_json_body(response: httpx.Response) -> Mapping[str, JsonValue]:
+    try:
+        return _JSON_OBJECT.validate_json(response.content)
+    except ValidationError:
+        raise BaseLLMException(message=response.text, status_code=response.status_code, headers=response.headers)
+
+
+def is_unsupported_disabled_thinking(model: str, thinking: object) -> bool:
+    return thinking == {"type": "disabled"} and litellm.utils.supports_factory(
+        model=model, custom_llm_provider="bigmodel", key="thinking_always_on"
+    )
+
+
+def bigmodel_business_error(
+    code: JsonValue, message: str, model: str, response: httpx.Response | None
+) -> ContextWindowExceededError | ContentPolicyViolationError | None:
+    match code:
+        case 1261 | "1261":
+            return ContextWindowExceededError(message=message, model=model, llm_provider="bigmodel", response=response)
+        case 1301 | "1301":
+            return ContentPolicyViolationError(message=message, model=model, llm_provider="bigmodel", response=response)
+        case _:
+            return None
+
+
+def raise_bigmodel_business_error(model: object, error_str: str, original_exception: object) -> None:
+    found: Final = _BUSINESS_CODE_IN_ERROR.search(error_str)
+    response: Final = getattr(original_exception, "response", None)
+    error: Final = bigmodel_business_error(
+        found.group(1) if found else None,
+        error_str,
+        str(model),
+        response if isinstance(response, httpx.Response) else None,
+    )
+    if error is not None:
+        raise error

litellm/llms/bigmodel/messages/transformation.py

@@ -0,0 +1,206 @@
+import json
+from collections.abc import AsyncIterator, Mapping, Sequence
+from typing import Final
+
+import httpx
+from openai._streaming import ServerSentEvent, SSEDecoder
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+from litellm.exceptions import AuthenticationError
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.llms.openai_like.messages.transformation import (
+    DEFAULT_ANTHROPIC_API_VERSION,
+    OpenAILikeAnthropicMessagesConfig,
+)
+from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
+from litellm.types.router import GenericLiteLLMParams
+
+from ..common_utils import (
+    bigmodel_json_body,
+    get_bigmodel_api_key,
+    get_bigmodel_url,
+    is_unsupported_disabled_thinking,
+)
+
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_OUTPUT_CONFIG: Final = TypeAdapter(dict[str, object])
+
+
+def _with_lowest_effort_thinking(
+    params: Mapping[str, object],
+) -> dict[str, object]:  # mutable-ok: the parent transform takes a request dict
+    requested_output_config: Final = params.get("output_config")
+    output_config: Final = {
+        **({} if "reasoning_effort" in params else {"effort": "low"}),
+        **(
+            _OUTPUT_CONFIG.validate_python(requested_output_config) if isinstance(requested_output_config, dict) else {}
+        ),
+    }
+    return {**params, "thinking": {"type": "adaptive"}, **({"output_config": output_config} if output_config else {})}
+
+
+def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]:  # mutable-ok: native JSON response
+    usage: Final = body.get("usage")
+    if not isinstance(usage, dict):
+        return dict(body)
+    server_tools: Final = usage.get("server_tool_use")
+    if not isinstance(server_tools, dict) or "web_fetch_requests" in server_tools:
+        return dict(body)
+    normalized: Final[JsonValue] = {
+        **body,
+        "usage": {**usage, "server_tool_use": {"web_fetch_requests": 0, **server_tools}},
+    }
+    return normalized
+
+
+class _BigModelMessagesStream(httpx.AsyncByteStream):
+    def __init__(self, response: httpx.Response) -> None:
+        self._response = response
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        async for event in SSEDecoder().aiter_bytes(self._response.aiter_bytes()):
+            yield _normalize_event(event)
+
+    async def aclose(self) -> None:
+        await self._response.aclose()
+
+
+def _json_object(data: str) -> Mapping[str, JsonValue] | None:
+    try:
+        return _JSON_OBJECT.validate_json(data)
+    except ValidationError:
+        return None
+
+
+def _normalized_data(data: str) -> str:
+    body: Final = _json_object(data)
+    if body is None:
+        return data
+    message: Final = body.get("message")
+    normalized: Final[JsonValue] = (
+        {**body, "message": _normalize_message(message)}
+        if body.get("type") == "message_start" and isinstance(message, dict)
+        else _normalize_message(body)
+    )
+    return data if normalized == body else json.dumps(normalized, ensure_ascii=False)
+
+
+def _normalize_event(event: ServerSentEvent) -> bytes:
+    metadata: Final = "".join(
+        f"{name}: {value}\n"
+        for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry))
+        if value is not None
+    )
+    data_lines: Final = "".join(f"data: {line}\n" for line in _normalized_data(event.data).split("\n"))
+    return f"{metadata}{data_lines}\n".encode()
+
+
+class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig):
+    @property
+    def custom_llm_provider(self) -> str:
+        return "bigmodel"
+
+    def is_native_messages_route(self) -> bool:
+        return True
+
+    def should_strip_billing_metadata(self) -> bool:
+        return True
+
+    def transform_anthropic_messages_request(
+        self,
+        model: str,
+        messages: list[dict[str, object]],  # mutable-ok: matches dict-typed base signature
+        anthropic_messages_optional_request_params: dict[str, object],  # mutable-ok: matches dict-typed base signature
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: matches dict-typed base signature
+    ) -> dict[str, object]:  # mutable-ok: matches dict-typed base signature
+        request_params: Final = (
+            _with_lowest_effort_thinking(anthropic_messages_optional_request_params)
+            if is_unsupported_disabled_thinking(model, anthropic_messages_optional_request_params.get("thinking"))
+            else anthropic_messages_optional_request_params
+        )
+        return super().transform_anthropic_messages_request(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the parent transform returns a bare dict
+            model, messages, request_params, litellm_params, headers
+        )
+
+    def transform_anthropic_messages_response(
+        self, model: str, raw_response: httpx.Response, logging_obj: Logging
+    ) -> AnthropicMessagesResponse:
+        normalized: Final = httpx.Response(
+            raw_response.status_code,
+            json=_normalize_message(bigmodel_json_body(raw_response)),
+            request=raw_response.request,
+        )
+        return super().transform_anthropic_messages_response(model, normalized, logging_obj)
+
+    def get_async_streaming_response_iterator(
+        self,
+        model: str,
+        httpx_response: httpx.Response,
+        request_body: dict[str, object],  # mutable-ok: inherited native stream API takes a request dict
+        litellm_logging_obj: Logging,
+    ) -> AsyncIterator[bytes]:
+        normalized: Final = httpx.Response(
+            httpx_response.status_code,
+            headers={
+                key: value
+                for key, value in httpx_response.headers.items()
+                if key.lower() not in ("content-encoding", "content-length")
+            },
+            stream=_BigModelMessagesStream(httpx_response),
+            request=httpx_response.request,
+        )
+        return super().get_async_streaming_response_iterator(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # native parent yields bytes but its iterator annotation omits the element type
+            model, normalized, request_body, litellm_logging_obj
+        )
+
+    @staticmethod
+    def _translate_reasoning_effort_to_anthropic(
+        model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str
+    ) -> None:
+        return None
+
+    @staticmethod
+    def _translate_adaptive_effort_for_non_adaptive_model(
+        model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str
+    ) -> None:
+        return None
+
+    @staticmethod
+    def _drop_incompatible_temperature_for_thinking(
+        model: str, optional_params: Mapping[str, object], custom_llm_provider: str
+    ) -> None:
+        return None
+
+    def validate_anthropic_messages_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[object],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> tuple[dict[str, str], str | None]:  # mutable-ok: HTTP handler merges forwarded headers
+        resolved_key: Final = get_bigmodel_api_key(api_key)
+        if resolved_key is None:
+            raise AuthenticationError(
+                message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
+            )
+        return {
+            **{key: value for key, value in headers.items() if key.lower() not in ("x-api-key", "authorization")},
+            "x-api-key": resolved_key,
+            "anthropic-version": DEFAULT_ANTHROPIC_API_VERSION,
+            "content-type": "application/json",
+        }, api_base
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        return get_bigmodel_url(api_base, "messages")

litellm/llms/bigmodel/responses/transformation.py

@@ -0,0 +1,263 @@
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
+import httpx
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
+from typing_extensions import Never
+
+from litellm.exceptions import AuthenticationError, UnsupportedParamsError
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.types.llms.openai import (
+    ResponseInputParam,
+    ResponsesAPIOptionalRequestParams,
+    ResponsesAPIResponse,
+    ResponsesAPIStreamingResponse,
+)
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+
+from ..common_utils import bigmodel_business_error, bigmodel_json_body, get_bigmodel_api_key, get_bigmodel_url
+
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_SUPPORTED_PARAMS: Final = (
+    "input",
+    "model",
+    "instructions",
+    "stream",
+    "temperature",
+    "top_p",
+    "max_output_tokens",
+    "tools",
+    "tool_choice",
+    "parallel_tool_calls",
+    "include",
+    "reasoning",
+    "text",
+    "prompt_cache_key",
+    "previous_response_id",
+    "store",
+    "extra_headers",
+    "extra_query",
+    "extra_body",
+    "timeout",
+)
+
+
+_STATUS_BY_BUSINESS_CODE: Final = MappingProxyType(
+    {
+        **dict.fromkeys((1000, 1001, 1003, 1005), 401),
+        **dict.fromkeys((1210, 1211, 1212, 1213, 1214, 1215, 1221, 1222, 1261, 1301), 400),
+        1220: 403,
+        **dict.fromkeys(
+            (1113, 1302, 1305, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321),
+            429,
+        ),
+    }
+)
+
+
+class _RequestedTools(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    tools: JsonValue = None
+    tool_choice: JsonValue = None
+    parallel_tool_calls: JsonValue = None
+
+
+def _requested_tools(logging_obj: Logging) -> _RequestedTools:
+    try:
+        return _RequestedTools.model_validate(logging_obj.model_call_details.get("optional_params") or {})
+    except ValidationError:
+        return _RequestedTools()
+
+
+def _envelope_status_code(code: JsonValue) -> int:
+    if not isinstance(code, int) or isinstance(code, bool):
+        return 500
+    if 400 <= code <= 599:
+        return code
+    return _STATUS_BY_BUSINESS_CODE.get(code, 500)
+
+
+def _normalize_output_item(item: JsonValue) -> JsonValue:
+    if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"):
+        return item
+    content: Final = item.get("content")
+    normalized: Final[JsonValue] = {"summary": [], **item} if item["type"] == "reasoning" else item
+    if not isinstance(content, dict):
+        return normalized
+    return {**normalized, "content": [content]}
+
+
+def _normalize_response(
+    body: Mapping[str, JsonValue],
+    requested: _RequestedTools,
+) -> dict[str, JsonValue]:  # mutable-ok: JSON serialization boundary
+    usage: Final = body.get("usage")
+    output: Final = body.get("output")
+    tools: Final = body.get("tools")
+    tool_choice: Final = body.get("tool_choice")
+    parallel_tool_calls: Final = body.get("parallel_tool_calls")
+    input_tokens: Final = usage.get("input_tokens") if isinstance(usage, dict) else None
+    output_tokens: Final = usage.get("output_tokens") if isinstance(usage, dict) else None
+    normalized_usage: Final = (
+        {
+            "total_tokens": input_tokens + output_tokens,
+            **usage,
+        }
+        if isinstance(usage, dict) and isinstance(input_tokens, int) and isinstance(output_tokens, int)
+        else usage
+    )
+    normalized: Final[JsonValue] = {
+        **body,
+        "parallel_tool_calls": (
+            requested.parallel_tool_calls is not False if parallel_tool_calls is None else parallel_tool_calls
+        ),
+        "tools": (requested.tools or []) if tools is None else tools,
+        "tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice,
+        **({"usage": normalized_usage} if "usage" in body else {}),
+        **({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}),
+    }
+    return normalized
+
+
+def _normalize_stream_part(part: JsonValue) -> JsonValue:
+    if not isinstance(part, dict):
+        return part
+    if part.get("type") == "output_text":
+        return {
+            "annotations": [],
+            "logprobs": None,
+            **part,
+        }
+    if part.get("type") == "reasoning_text":
+        return {"reasoning": part.get("text", ""), **part}
+    return part
+
+
+class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig):
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.BIGMODEL
+
+    def supports_native_file_search(self) -> bool:
+        return False
+
+    def supports_native_websocket(self) -> bool:
+        return False
+
+    def should_fake_stream(
+        self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None
+    ) -> bool:
+        return False
+
+    def may_answer_stream_with_json(self) -> bool:
+        return True
+
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: inherited API requires a list
+        return list(_SUPPORTED_PARAMS)
+
+    def map_openai_params(
+        self,
+        response_api_optional_params: ResponsesAPIOptionalRequestParams,
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: request pipeline adds parameters to the returned body
+        return {key: value for key, value in response_api_optional_params.items() if key in _SUPPORTED_PARAMS}
+
+    def validate_environment(
+        self, headers: Mapping[str, str], model: str, litellm_params: GenericLiteLLMParams | None
+    ) -> dict[str, str]:  # mutable-ok: HTTP handler merges forwarded headers
+        resolved_key: Final = get_bigmodel_api_key(litellm_params.api_key if litellm_params else None)
+        if resolved_key is None:
+            raise AuthenticationError(
+                message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
+            )
+        return {
+            **headers,
+            "Authorization": f"Bearer {resolved_key}",
+            "Content-Type": "application/json",
+        }
+
+    def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str:
+        return get_bigmodel_url(api_base, "responses")
+
+    def transform_response_api_response(
+        self, model: str, raw_response: httpx.Response, logging_obj: Logging
+    ) -> ResponsesAPIResponse:
+        body: Final = bigmodel_json_body(raw_response)
+        if body.get("success") is False:
+            message: Final = str(body.get("msg") or raw_response.text)
+            raise bigmodel_business_error(body.get("code"), message, model, None) or BaseLLMException(
+                message=message,
+                status_code=_envelope_status_code(body.get("code")),
+                headers=raw_response.headers,
+            )
+        normalized: Final = _normalize_response(body, _requested_tools(logging_obj))
+        normalized_response: Final = httpx.Response(
+            status_code=raw_response.status_code,
+            headers={
+                key: value
+                for key, value in raw_response.headers.items()
+                if key.lower() not in ("content-encoding", "content-length")
+            },
+            json=normalized,
+            request=raw_response.request,
+        )
+        return super().transform_response_api_response(model, normalized_response, logging_obj)
+
+    def transform_get_response_api_response(
+        self, raw_response: httpx.Response, logging_obj: Logging
+    ) -> ResponsesAPIResponse:
+        return self.transform_response_api_response("", raw_response, logging_obj)
+
+    def transform_streaming_response(
+        self, model: str, parsed_chunk: Mapping[str, object], logging_obj: Logging
+    ) -> ResponsesAPIStreamingResponse:
+        chunk: Final = _JSON_OBJECT.validate_python(parsed_chunk)
+        response: Final = chunk.get("response")
+        item: Final = chunk.get("item")
+        normalized: Final[JsonValue] = {
+            **chunk,
+            **(
+                {"response": _normalize_response(response, _requested_tools(logging_obj))}
+                if isinstance(response, dict)
+                else {}
+            ),
+            **({"item": _normalize_output_item(item)} if "item" in chunk else {}),
+        }
+        event_model: Final = self.get_event_model_class(str(normalized.get("type")))
+        match chunk.get("type"):
+            case "response.output_text.done":
+                return event_model.model_validate({"logprobs": [], **normalized})
+            case "response.content_part.done":
+                return event_model.model_validate({**normalized, "part": _normalize_stream_part(chunk.get("part"))})
+            case "error":
+                error: Final = chunk.get("error")
+                error_fields: Final = error if isinstance(error, dict) else chunk
+                normalized_error: Final[JsonValue] = {
+                    **normalized,
+                    "error": {"type": "api_error", **error_fields, "code": error_fields.get("code") or "unknown_error"},
+                }
+                return event_model.model_validate(normalized_error)
+            case _:
+                return event_model.model_validate(normalized)
+
+    def transform_cancel_response_api_request(
+        self, response_id: str, api_base: str, litellm_params: GenericLiteLLMParams, headers: Mapping[str, str]
+    ) -> Never:
+        raise UnsupportedParamsError(message="BigModel does not support cancelling responses", status_code=400)
+
+    def transform_compact_response_api_request(
+        self,
+        model: str,
+        input: str | ResponseInputParam,
+        response_api_optional_request_params: Mapping[str, object],
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> Never:
+        raise UnsupportedParamsError(message="BigModel does not support compacting responses", status_code=400)

litellm/llms/custom_httpx/http_handler.py

@@ -764,18 +764,23 @@ class AsyncHTTPHandler:
         follow_redirects: bool | None = None,
         timeout: float | httpx.Timeout | None = None,
         max_response_bytes: int | None = None,
+        stream: bool = False,
     ):
         # Set follow_redirects to UseClientDefault if None
         _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
+        query: Final = httpx.QueryParams(params).merge(HTTPHandler.extract_query_params(url))
+        request_headers: Final = httpx.Headers(headers)
 
-        params = params or {}
-        params.update(HTTPHandler.extract_query_params(url))
+        if stream:
+            return await self._send_streaming_get(
+                url, params=query, headers=request_headers, follow_redirects=follow_redirects, timeout=timeout
+            )
 
         if max_response_bytes is not None:
             return await self._get_with_response_limit(
                 url,
-                params=httpx.QueryParams(params),
-                headers=httpx.Headers(headers),
+                params=query,
+                headers=request_headers,
                 max_bytes=max_response_bytes,
                 follow_redirects=self.client.follow_redirects if follow_redirects is None else follow_redirects,
                 timeout=self.client.timeout if timeout is None else httpx.Timeout(timeout),
@@ -783,13 +788,41 @@ class AsyncHTTPHandler:
 
         response: Final = await self.client.get(
             url,
-            params=params,
-            headers=headers,
+            params=query,
+            headers=request_headers,
             follow_redirects=_follow_redirects,
             timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT,
         )
         return response
 
+    async def _send_streaming_get(
+        self,
+        url: str,
+        *,
+        params: httpx.QueryParams,
+        headers: httpx.Headers,
+        follow_redirects: bool | None,
+        timeout: float | httpx.Timeout | None,
+    ) -> httpx.Response:
+        start_time: Final = time.time()
+        request: Final = self.client.build_request(
+            "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout
+        )
+        try:
+            response: Final = await self.client.send(
+                request,
+                stream=True,
+                follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects,
+            )
+            _anchor_handler_to(response, self)
+            response.raise_for_status()
+        except httpx.TimeoutException as e:
+            _raise_timeout(e, start_time)
+        except httpx.HTTPStatusError as e:
+            await _raise_masked_async_error(e, True)
+        else:
+            return response
+
     async def _get_with_response_limit(
         self,
         url: str,
@@ -1468,15 +1501,20 @@ class HTTPHandler:
         headers: Mapping[str, Any] | None = None,
         follow_redirects: bool | None = None,
         timeout: float | httpx.Timeout | None = None,
+        stream: bool = False,
     ):
         # Set follow_redirects to UseClientDefault if None
         _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
-        params = params or {}
-        params.update(self.extract_query_params(url))
+        query: Final = httpx.QueryParams(params).merge(self.extract_query_params(url))
+
+        if stream:
+            return self._send_streaming_get(
+                url, params=query, headers=headers, follow_redirects=follow_redirects, timeout=timeout
+            )
 
         response: Final = self.client.get(
             url,
-            params=params,
+            params=query,
             headers=headers,
             follow_redirects=_follow_redirects,
             timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT,
@@ -1484,6 +1522,34 @@ class HTTPHandler:
 
         return response
 
+    def _send_streaming_get(
+        self,
+        url: str,
+        *,
+        params: httpx.QueryParams,
+        headers: Mapping[str, str] | None,
+        follow_redirects: bool | None,
+        timeout: float | httpx.Timeout | None,
+    ) -> httpx.Response:
+        start_time: Final = time.time()
+        request: Final = self.client.build_request(
+            "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout
+        )
+        try:
+            response: Final = self.client.send(
+                request,
+                stream=True,
+                follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects,
+            )
+            _anchor_handler_to(response, self)
+            response.raise_for_status()
+        except httpx.TimeoutException as e:
+            _raise_timeout(e, start_time)
+        except httpx.HTTPStatusError as e:
+            _raise_masked_sync_error(e, True)
+        else:
+            return response
+
     @staticmethod
     def extract_query_params(url: str) -> dict[str, str]:
         """

litellm/llms/custom_httpx/llm_http_handler.py

@@ -367,12 +367,37 @@ class _SignedBodyKwargs(TypedDict, total=False):
     json: ReadOnly[dict[str, object]]
 
 
+def _is_sized_json_body(response: httpx.Response) -> bool:
+    headers: Final = httpx.Headers(response.headers)
+    media_types: Final = (value.split(";")[0].strip().lower() for value in headers.get_list("content-type"))
+    return "content-length" in headers and "application/json" in media_types
+
+
+def _is_json_document(content: bytes) -> bool:
+    try:
+        return isinstance(json.loads(content), dict)
+    except ValueError:
+        return False
+
+
 def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -> _SignedBodyKwargs:
     if signed_body is not None:
         return {"data": signed_body}
     return {"json": data}
 
 
+def _answered_json(response: httpx.Response) -> bool:
+    return "application/json" in response.headers.get("content-type", "").lower()
+
+
+def _stream_query_params(stream: bool, starting_after: int | None) -> Mapping[str, object]:
+    if not stream:
+        return {}
+    if starting_after is None:
+        return {"stream": "true"}
+    return {"stream": "true", "starting_after": starting_after}
+
+
 def _google_genai_streaming_hidden_params(
     *,
     api_base: str,
@@ -3025,6 +3050,17 @@ class BaseLLMHTTPHandler:
             },
         )
 
+        replay_as_stream: Final = partial(
+            MockResponsesAPIStreamingIterator,
+            model=model,
+            logging_obj=logging_obj,
+            responses_api_provider_config=responses_api_provider_config,
+            litellm_metadata=litellm_metadata,
+            custom_llm_provider=custom_llm_provider,
+            request_data=request_context,
+            call_type=CallTypes.responses.value,
+        )
+
         try:
             if is_stream_request:
                 response = sync_httpx_client.post(
@@ -3036,7 +3072,13 @@ class BaseLLMHTTPHandler:
                 )
                 if caller_requested_stream:
                     if fake_stream is True:
-                        return MockResponsesAPIStreamingIterator(
+                        return replay_as_stream(response=response)
+                    if not (
+                        responses_api_provider_config.may_answer_stream_with_json()
+                        and _is_sized_json_body(response)
+                        and _is_json_document(response.read())
+                    ):
+                        return SyncResponsesAPIStreamingIterator(
                             response=response,
                             model=model,
                             logging_obj=logging_obj,
@@ -3046,17 +3088,6 @@ class BaseLLMHTTPHandler:
                             request_data=request_context,
                             call_type=CallTypes.responses.value,
                         )
-
-                    return SyncResponsesAPIStreamingIterator(
-                        response=response,
-                        model=model,
-                        logging_obj=logging_obj,
-                        responses_api_provider_config=responses_api_provider_config,
-                        litellm_metadata=litellm_metadata,
-                        custom_llm_provider=custom_llm_provider,
-                        request_data=request_context,
-                        call_type=CallTypes.responses.value,
-                    )
                 response.read()
             else:
                 response = sync_httpx_client.post(
@@ -3071,6 +3102,9 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if is_stream_request and caller_requested_stream:
+            return replay_as_stream(response=response)
+
         initial_response: Final = responses_api_provider_config.transform_response_api_response(
             model=model,
             raw_response=response,
@@ -3226,6 +3260,17 @@ class BaseLLMHTTPHandler:
             },
         )
 
+        replay_as_stream: Final = partial(
+            MockResponsesAPIStreamingIterator,
+            model=model,
+            logging_obj=logging_obj,
+            responses_api_provider_config=responses_api_provider_config,
+            litellm_metadata=litellm_metadata,
+            custom_llm_provider=custom_llm_provider,
+            request_data=request_context,
+            call_type=CallTypes.responses.value,
+        )
+
         try:
             if is_stream_request:
                 response = await async_httpx_client.post(
@@ -3239,7 +3284,13 @@ class BaseLLMHTTPHandler:
 
                 if caller_requested_stream:
                     if fake_stream is True:
-                        return MockResponsesAPIStreamingIterator(
+                        return replay_as_stream(response=response)
+                    if not (
+                        responses_api_provider_config.may_answer_stream_with_json()
+                        and _is_sized_json_body(response)
+                        and _is_json_document(await response.aread())
+                    ):
+                        return ResponsesAPIStreamingIterator(
                             response=response,
                             model=model,
                             logging_obj=logging_obj,
@@ -3249,17 +3300,6 @@ class BaseLLMHTTPHandler:
                             request_data=request_context,
                             call_type=CallTypes.responses.value,
                         )
-
-                    return ResponsesAPIStreamingIterator(
-                        response=response,
-                        model=model,
-                        logging_obj=logging_obj,
-                        responses_api_provider_config=responses_api_provider_config,
-                        litellm_metadata=litellm_metadata,
-                        custom_llm_provider=custom_llm_provider,
-                        request_data=request_context,
-                        call_type=CallTypes.responses.value,
-                    )
                 await response.aread()
             else:
                 response = await async_httpx_client.post(
@@ -3276,6 +3316,9 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if is_stream_request and caller_requested_stream:
+            return replay_as_stream(response=response)
+
         initial_response: Final = responses_api_provider_config.transform_response_api_response(
             model=model,
             raw_response=response,
@@ -3493,12 +3536,20 @@ class BaseLLMHTTPHandler:
         logging_obj: LiteLLMLoggingObj,
         custom_llm_provider: str | None = None,
         extra_headers: dict[str, object] | None = None,
+        extra_query: Mapping[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
+        stream: bool = False,
+        starting_after: int | None = None,
+        litellm_metadata: Mapping[str, object] = MappingProxyType({}),
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         _is_async: bool = False,
         shared_session: Optional["ClientSession"] = None,
-    ) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]:
+    ) -> (
+        ResponsesAPIResponse
+        | SyncResponsesAPIStreamingIterator
+        | Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator]
+    ):
         """
         Get a response by ID
         Uses GET /v1/responses/{response_id} endpoint in the responses API
@@ -3511,7 +3562,11 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
                 custom_llm_provider=custom_llm_provider,
                 extra_headers=extra_headers,
+                extra_query=extra_query,
                 extra_body=extra_body,
+                stream=stream,
+                starting_after=starting_after,
+                litellm_metadata=litellm_metadata,
                 timeout=timeout,
                 client=client,
                 shared_session=shared_session,
@@ -3540,20 +3595,23 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             headers=headers,
         )
+        query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)}
 
         ## LOGGING
         logging_obj.pre_call(
             input="",
             api_key="",
             additional_args={
-                "complete_input_dict": data,
+                "complete_input_dict": query,
                 "api_base": api_base,
                 "headers": headers,
             },
         )
 
         try:
-            response: Final = sync_httpx_client.get(url=url, headers=headers, params=data)
+            response: Final = sync_httpx_client.get(
+                url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None
+            )
             response.raise_for_status()
         except Exception as e:
             raise self._handle_error(
@@ -3561,6 +3619,20 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if stream and not _answered_json(response):
+            return SyncResponsesAPIStreamingIterator(
+                response=response,
+                model="",
+                logging_obj=logging_obj,
+                responses_api_provider_config=responses_api_provider_config,
+                litellm_metadata=dict(litellm_metadata),
+                custom_llm_provider=custom_llm_provider,
+                request_data={"litellm_params": dict(litellm_params)},
+                replays_stored_stream=True,
+            )
+
+        if stream:
+            response.read()
         return responses_api_provider_config.transform_get_response_api_response(
             raw_response=response,
             logging_obj=logging_obj,
@@ -3574,11 +3646,15 @@ class BaseLLMHTTPHandler:
         logging_obj: LiteLLMLoggingObj,
         custom_llm_provider: str | None = None,
         extra_headers: dict[str, object] | None = None,
+        extra_query: Mapping[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
+        stream: bool = False,
+        starting_after: int | None = None,
+        litellm_metadata: Mapping[str, object] = MappingProxyType({}),
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         shared_session: Optional["ClientSession"] = None,
-    ) -> ResponsesAPIResponse:
+    ) -> ResponsesAPIResponse | ResponsesAPIStreamingIterator:
         """
         Async version of get_responses
         """
@@ -3616,20 +3692,23 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             headers=headers,
         )
+        query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)}
 
         ## LOGGING
         logging_obj.pre_call(
             input="",
             api_key="",
             additional_args={
-                "complete_input_dict": data,
+                "complete_input_dict": query,
                 "api_base": api_base,
                 "headers": headers,
             },
         )
 
         try:
-            response: Final = await async_httpx_client.get(url=url, headers=headers, params=data)
+            response: Final = await async_httpx_client.get(
+                url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None
+            )
             response.raise_for_status()
         except Exception as e:
             verbose_logger.debug("Error retrieving response: %s", e)
@@ -3638,6 +3717,20 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if stream and not _answered_json(response):
+            return ResponsesAPIStreamingIterator(
+                response=response,
+                model="",
+                logging_obj=logging_obj,
+                responses_api_provider_config=responses_api_provider_config,
+                litellm_metadata=dict(litellm_metadata),
+                custom_llm_provider=custom_llm_provider,
+                request_data={"litellm_params": dict(litellm_params)},
+                replays_stored_stream=True,
+            )
+
+        if stream:
+            await response.aread()
         return responses_api_provider_config.transform_get_response_api_response(
             raw_response=response,
             logging_obj=logging_obj,

litellm/llms/github_copilot/authenticator.py

@@ -1,11 +1,12 @@
 import json
 import os
 import time
+from dataclasses import dataclass
 from datetime import datetime
-from typing import Any, Final
+from typing import Any, Final, TypeVar
 
 import httpx
-from pydantic import ConfigDict, TypeAdapter, with_config
+from pydantic import BaseModel, ConfigDict, TypeAdapter, with_config
 from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm._logging import verbose_logger
@@ -13,7 +14,6 @@ from litellm.litellm_core_utils.asyncify import can_block_current_thread
 from litellm.llms.custom_httpx.http_handler import get_httpx_client
 
 from .common_utils import (
-    APIKeyExpiredError,
     GetAccessTokenError,
     GetAPIKeyError,
     GetDeviceCodeError,
@@ -27,6 +27,44 @@ DEFAULT_GITHUB_ACCESS_TOKEN_URL: Final = "https://github.com/login/oauth/access_
 DEFAULT_GITHUB_API_KEY_URL: Final = "https://api.github.com/copilot_internal/v2/token"
 
 
+class CopilotEndpoints(BaseModel):
+    model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True)
+
+    api: str | None = None
+
+
+class CopilotCachedKey(BaseModel):
+    model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True)
+
+    token: str
+    expires_at: float
+    endpoints: CopilotEndpoints | None = None
+
+
+class CopilotCachedEndpoints(BaseModel):
+    model_config = ConfigDict(frozen=True, strict=True, extra="ignore", hide_input_in_errors=True)
+
+    endpoints: CopilotEndpoints | None = None
+
+
+ApiKeyFileView = TypeVar("ApiKeyFileView", CopilotCachedKey, CopilotCachedEndpoints)
+
+
+@dataclass(frozen=True, slots=True)
+class _UnsavedApiKey:
+    access_token: str
+    key: CopilotCachedKey
+
+
+_UNSAVED_API_KEYS: Final[dict[str, _UnsavedApiKey]] = {}  # mutable-ok: refreshed keys api-key.json could not hold
+
+
+def _read_token_file(path: str) -> str:
+    with open(path, "rb") as f:
+        raw: Final = f.read()
+    return raw.decode(json.detect_encoding(raw))
+
+
 @with_config(ConfigDict(extra="allow", strict=True))
 class _DeviceCode(TypedDict):
     device_code: ReadOnly[str]
@@ -87,12 +125,13 @@ class Authenticator:
             GetAccessTokenError: If unable to obtain an access token after retries.
         """
         try:
-            with open(self.access_token_file, "r") as f:
-                access_token = f.read().strip()
-                if access_token:
-                    return access_token
+            saved_token: Final = _read_token_file(self.access_token_file).strip()
+            if saved_token:
+                return saved_token
         except OSError:
             verbose_logger.warning("No existing access token found or error reading file")
+        except ValueError as e:
+            verbose_logger.warning("Ignoring %s, not a text access token: %s", self.access_token_file, e)
 
         if not can_block_current_thread():
             raise GetAccessTokenError(
@@ -135,64 +174,83 @@ class Authenticator:
         Raises:
             GetAPIKeyError: If unable to obtain an API key.
         """
-        try:
-            with open(self.api_key_file, "r") as f:
-                api_key_info = json.load(f)
-                if api_key_info.get("expires_at", 0) > datetime.now().timestamp():
-                    return api_key_info.get("token")
-                else:
-                    verbose_logger.warning("API key expired, refreshing")
-                    raise APIKeyExpiredError(
-                        message="API key expired",
-                        status_code=401,
-                    )
-        except OSError:
-            verbose_logger.warning("No API key file found or error opening file")
-        except (json.JSONDecodeError, KeyError) as e:
-            verbose_logger.warning("Error reading API key from file: %s", e)
-        except APIKeyExpiredError:
-            pass  # Already logged in the try block
+        unsaved: Final = self._unsaved_api_key()
+        if unsaved is not None:
+            return unsaved.token
+        cached: Final = self._read_api_key_file(CopilotCachedKey)
+        if cached is not None and cached.expires_at > datetime.now().timestamp():
+            return cached.token
+        if cached is not None:
+            verbose_logger.warning("API key expired, refreshing")
 
         try:
-            api_key_info = self._refresh_api_key()
-            with open(self.api_key_file, "w") as f:
-                json.dump(api_key_info, f)
-            token: Final = api_key_info.get("token")
-            if token:
-                return token
-            else:
-                raise GetAPIKeyError(
-                    message="API key response missing token",
-                    status_code=401,
-                )
-        except OSError as e:
-            verbose_logger.error("Error saving API key to file: %s", e)
-            raise GetAPIKeyError(
-                message=f"Failed to save API key: {e}",
-                status_code=500,
-            )
+            api_key_info: Final = self._refresh_api_key()
         except RefreshAPIKeyError as e:
             raise GetAPIKeyError(
                 message=f"Failed to refresh API key: {e}",
                 status_code=401,
+            ) from e
+        serialized: Final = json.dumps(api_key_info)
+        try:
+            with open(self.api_key_file, "w") as f:
+                f.write(serialized)
+        except OSError as e:
+            verbose_logger.warning(
+                "Keeping the refreshed API key in memory, it could not be saved to %s: %s", self.api_key_file, e
             )
+            self._keep_unsaved_api_key(serialized)
+        token: Final = api_key_info.get("token")
+        if not token:
+            raise GetAPIKeyError(
+                message="API key response missing token",
+                status_code=401,
+            )
+        return token
 
     def get_api_base(self) -> str | None:
         """
-        Get the API endpoint from the api-key.json file.
+        Get the API endpoint of the current API key.
 
         Returns:
             Optional[str]: The GitHub Copilot API endpoint, or None if not found.
         """
-        try:
-            with open(self.api_key_file, "r") as f:
-                api_key_info: Final = json.load(f)
-                endpoints: Final = api_key_info.get("endpoints", {})
-                api_endpoint: Final = endpoints.get("api")
-                return api_endpoint
-        except (OSError, json.JSONDecodeError, KeyError) as e:
-            verbose_logger.warning("Error reading API endpoint from file: %s", e)
+        cached: Final = self._unsaved_api_key() or self._read_api_key_file(CopilotCachedEndpoints)
+        if cached is None or cached.endpoints is None:
             return None
+        return cached.endpoints.api
+
+    def _read_api_key_file(self, view: type[ApiKeyFileView]) -> ApiKeyFileView | None:
+        try:
+            return view.model_validate_json(_read_token_file(self.api_key_file))
+        except OSError:
+            verbose_logger.warning("No API key file found or error opening file")
+            return None
+        except ValueError as e:
+            verbose_logger.warning("Ignoring %s, not a Copilot API key file: %s", self.api_key_file, e)
+            return None
+
+    def _stored_access_token(self) -> str | None:
+        try:
+            access_token: Final = _read_token_file(self.access_token_file).strip()
+        except (OSError, ValueError):
+            return None
+        return access_token or None
+
+    def _unsaved_api_key(self) -> CopilotCachedKey | None:
+        unsaved: Final = _UNSAVED_API_KEYS.get(self.api_key_file)
+        if unsaved is None or unsaved.key.expires_at <= time.time():
+            return None
+        return unsaved.key if unsaved.access_token == self._stored_access_token() else None
+
+    def _keep_unsaved_api_key(self, serialized: str) -> None:
+        access_token: Final = self._stored_access_token()
+        if access_token is None:
+            return
+        try:
+            key: Final = CopilotCachedKey.model_validate_json(serialized)
+        except ValueError:
+            return
+        _UNSAVED_API_KEYS[self.api_key_file] = _UnsavedApiKey(access_token=access_token, key=key)
 
     def _refresh_api_key(self) -> dict[str, Any]:
         """

litellm/llms/manus/responses/transformation.py

@@ -241,7 +241,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         """
         Transform the get response API request into a URL and data.
 
@@ -252,7 +252,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
         """
         encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
         url: Final = f"{api_base}/{encoded_response_id}"
-        data: Final[dict] = {}
+        data: Final[dict[str, str]] = {}
         return url, data
 
     def transform_get_response_api_response(

litellm/llms/openai/common_utils.py

@@ -31,7 +31,7 @@ if TYPE_CHECKING:
 import litellm
 from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
-from litellm.litellm_core_utils.token_counter import token_counter
+from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
     _DEFAULT_TTL_FOR_HTTPX_CLIENTS,
@@ -242,7 +242,7 @@ async def _aiter_once(chunk: ChatCompletionChunk) -> AsyncIterator[ChatCompletio
 
 
 def build_output_token_limit_response(
-    e: openai.BadRequestError, data: Mapping[str, object], is_async: bool
+    e: openai.BadRequestError, data: Mapping[str, object], is_async: bool, base_model: str | None
 ) -> tuple[httpx.Headers, ChatCompletion | Iterator[ChatCompletionChunk] | AsyncIterator[ChatCompletionChunk]]:
     """Synthesize the length-truncated response the provider itself returns for slightly larger output budgets.
 
@@ -251,7 +251,11 @@ def build_output_token_limit_response(
     """
     model: Final[str] = str(data.get("model", ""))
     messages: Final = data.get("messages")
-    prompt_tokens: Final = token_counter(model=model, messages=messages) if isinstance(messages, list) else 0
+    prompt_tokens: Final = (
+        token_counter(model=usage_tokenizer_model(model, base_model), messages=messages)
+        if isinstance(messages, list)
+        else 0
+    )
     if not data.get("stream"):
         return e.response.headers, _output_token_limit_completion(model, prompt_tokens)
     chunk: Final = _output_token_limit_chunk(model)

litellm/llms/openai/openai.py

@@ -44,6 +44,7 @@ from litellm.utils import (
     CustomStreamWrapper,
     ProviderConfigManager,
     convert_to_model_response_object,
+    get_base_model_from_metadata,
 )
 
 from ...types.llms.openai import *
@@ -557,7 +558,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         except openai.BadRequestError as e:
             if not is_output_token_limit_error(e):
                 raise
-            return build_output_token_limit_response(e=e, data=data, is_async=True)
+            return build_output_token_limit_response(
+                e=e,
+                data=data,
+                is_async=True,
+                base_model=get_base_model_from_metadata(logging_obj.model_call_details),
+            )
         except Exception as e:
             raise e
 
@@ -604,7 +610,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         except openai.BadRequestError as e:
             if not is_output_token_limit_error(e):
                 raise
-            return build_output_token_limit_response(e=e, data=data, is_async=False)
+            return build_output_token_limit_response(
+                e=e,
+                data=data,
+                is_async=False,
+                base_model=get_base_model_from_metadata(logging_obj.model_call_details),
+            )
         except Exception as e:
             if raw_response is not None:
                 raise Exception(

litellm/llms/openai/realtime/handler.py

@@ -4,11 +4,17 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint.
 This requires websockets, and is currently only supported on LiteLLM Proxy.
 """
 
+import asyncio
 import ssl
-from typing import TYPE_CHECKING, Final, cast
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final, Literal, cast
+
+from pydantic import TypeAdapter
 
 from litellm._logging import redact_string, verbose_logger
 from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
+from litellm.exceptions import AuthenticationError
+from litellm.responses.streaming_iterator import map_stream_error_to_exception
 from litellm.types.realtime import RealtimeQueryParams
 
 from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
@@ -25,6 +31,54 @@ from ..openai import OpenAIChatCompletion
 
 if TYPE_CHECKING:
     from fastapi import WebSocket
+    from openai.types.realtime import RealtimeError
+    from websockets.asyncio.client import ClientConnection
+
+_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object])
+_CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model"
+_CREDENTIALS_REJECTED: Final = "invalid_api_key"
+
+
+def first_event_error(first_event: str | bytes) -> "RealtimeError | None":
+    event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event)
+    if event.get("type") != "error":
+        return None
+    from openai.types.realtime import RealtimeErrorEvent
+
+    return RealtimeErrorEvent.model_validate(event).error
+
+
+def _confirm_first_event(
+    first_event: str | bytes, model: str, sent_credentials: bool, model_listed_outside_realtime: bool
+) -> Literal[True]:
+    error: Final = first_event_error(first_event)
+    if error is None or (
+        error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime
+    ):
+        return True
+    if error.code == _CREDENTIALS_REJECTED or not sent_credentials:
+        raise AuthenticationError(message=error.message, llm_provider="openai", model=model)
+    raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai")
+
+
+async def confirm_session_started(
+    connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool
+) -> Literal[True]:
+    return _confirm_first_event(await connection.recv(), model, sent_credentials, model_listed_outside_realtime)
+
+
+async def confirm_gateway_session_started(
+    connection: "ClientConnection",
+    model: str,
+    sent_credentials: bool,
+    model_listed_outside_realtime: bool,
+    first_event_wait_seconds: float,
+) -> Literal[True]:
+    try:
+        first_event: Final = await asyncio.wait_for(connection.recv(), first_event_wait_seconds)
+    except asyncio.TimeoutError:
+        return True
+    return _confirm_first_event(first_event, model, sent_credentials, model_listed_outside_realtime)
 
 
 class OpenAIRealtime(OpenAIChatCompletion):

litellm/llms/openai/responses/transformation.py

@@ -1,4 +1,4 @@
-from collections.abc import Callable, Collection, Mapping, Sequence
+from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
 from functools import lru_cache
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints
@@ -10,7 +10,11 @@ from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api
+from litellm.integrations.anthropic_cache_control_hook import (
+    AnthropicCacheControlHook,
+    openai_prompt_cache_breakpoint_support,
+    targets_openai_api,
+)
 from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers
 from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
 from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
@@ -20,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     cache_control_as_prompt_cache_breakpoint,
     drop_non_python_regex_patterns,
     flatten_combinators_and_drop_non_python_regex_patterns,
+    prompt_cache_breakpoint_for_wire,
     with_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@@ -66,6 +71,7 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE,
 _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS
 _ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"})
 _PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"})
+_INPUT_PART_LIST_KEYS: Final = frozenset({"content", "output"})
 _COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"})
 _COMPACT_REJECTED_PARAMS: Final = frozenset(
     {
@@ -93,6 +99,13 @@ _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str,
 _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]])
 
 
+class _DeclaredModel(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
+
+    base_model: object = None
+    model_info: object = None
+
+
 class _DeploymentHost(BaseModel):
     """`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`."""
 
@@ -216,6 +229,63 @@ def _with_breakpoint_on_last_reading_part(
     return (*parts[:last], marked_last, *parts[last + 1 :])
 
 
+def _json_object_or_none(value: object) -> Mapping[str, object] | None:
+    return value if isinstance(value, Mapping) else None
+
+
+def _json_array_or_empty(value: object) -> Sequence[object]:
+    return value if isinstance(value, (list, tuple)) else ()
+
+
+def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object:
+    fields: Final = _json_object_or_none(item)
+    if fields is None or not _carries_prompt_cache_breakpoint(fields):
+        return item
+    wire_safe_fields: Final = {
+        key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers)
+        for key, value in fields.items()
+        if key != "prompt_cache_breakpoint"
+    }
+    unchanged: Final = "prompt_cache_breakpoint" not in fields and _same_objects(
+        wire_safe_fields.values(), fields.values()
+    )
+    return item if unchanged else wire_safe_fields
+
+
+def _same_objects(rebuilt: Iterable[object], originals: Iterable[object]) -> bool:
+    return all(new is old for new, old in zip(rebuilt, originals, strict=True))
+
+
+def _carries_prompt_cache_breakpoint(fields: Mapping[str, object]) -> bool:
+    return "prompt_cache_breakpoint" in fields or any(
+        _parts_carry_prompt_cache_breakpoint(fields.get(key)) for key in _INPUT_PART_LIST_KEYS
+    )
+
+
+def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool:
+    return any(isinstance(part, Mapping) and "prompt_cache_breakpoint" in part for part in _json_array_or_empty(parts))
+
+
+def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object:
+    if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value):
+        return value
+    parts: Final = _json_array_or_empty(value)
+    wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in parts)
+    return value if _same_objects(wire_safe_parts, parts) else list(wire_safe_parts)
+
+
+def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object:
+    fields: Final = _json_object_or_none(part)
+    if fields is None or "prompt_cache_breakpoint" not in fields:
+        return part
+    marker: Final = fields["prompt_cache_breakpoint"]
+    wire_marker: Final = prompt_cache_breakpoint_for_wire(marker, drop_params=True) if keeps_markers else None
+    if keeps_markers and wire_marker == marker:
+        return part
+    unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
+    return with_prompt_cache_breakpoint(unmarked, wire_marker)
+
+
 class _ReasoningSupportEntry(LiteLLMBaseModel):
     litellm_provider: str | None = None
     supports_reasoning: bool | None = None
@@ -544,12 +614,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         validated_input: Final = self._validate_input_param(
             self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params)
         )
-        marked_input: Final = (
-            self._input_with_prompt_cache_breakpoints(validated_input)
-            if self._reads_prompt_cache_breakpoints(
-                model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options
-            )
-            else validated_input
+        reads_breakpoints: Final = self._reads_prompt_cache_breakpoints(
+            model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options
+        )
+        marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints(
+            model=model, input=validated_input, litellm_params=litellm_params, reads_breakpoints=reads_breakpoints
         )
         maps_anthropic_tool_search: Final = self._targets_openai_api(litellm_params) and any(
             is_anthropic_tool_search_chat_tool(tool) for tool in tools or ()
@@ -559,20 +628,71 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
             if maps_anthropic_tool_search
             else tools
         )
-        replayable_input: Final = (
-            input_without_unanswered_anthropic_tool_search_calls(marked_input)
-            if maps_anthropic_tool_search
-            else marked_input
-        )
         stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools(
-            model=model, input=replayable_input, tools=mapped_tools
+            model=model,
+            input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
+            tools=mapped_tools,
         )
+        if marker_safe_input is not validated_input:
+            self._strip_caller_input_as_if_unmarked(
+                model=model,
+                caller_input=validated_input,
+                reads_breakpoints=reads_breakpoints,
+                maps_anthropic_tool_search=maps_anthropic_tool_search,
+            )
         object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
         sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
             model=model, tools=object_schema_tools, litellm_params=litellm_params
         )
         return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools
 
+    def _replayable_input(
+        self, input: str | ResponseInputParam, reads_breakpoints: bool, maps_anthropic_tool_search: bool
+    ) -> str | ResponseInputParam:
+        marked_input: Final = self._input_with_prompt_cache_breakpoints(input) if reads_breakpoints else input
+        if not maps_anthropic_tool_search:
+            return marked_input
+        return input_without_unanswered_anthropic_tool_search_calls(marked_input)
+
+    def _strip_caller_input_as_if_unmarked(
+        self,
+        model: str,
+        caller_input: str | ResponseInputParam,
+        reads_breakpoints: bool,
+        maps_anthropic_tool_search: bool,
+    ) -> None:
+        self.remove_cache_control_flag_from_input_and_tools(
+            model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
+        )
+
+    def _input_with_wire_safe_prompt_cache_breakpoints(
+        self,
+        model: str,
+        input: str | ResponseInputParam,
+        litellm_params: GenericLiteLLMParams,
+        reads_breakpoints: bool,
+    ) -> str | ResponseInputParam:
+        if not isinstance(input, list) or not drops_params(litellm_params):
+            return input
+        keeps_markers: Final = reads_breakpoints or self._keeps_prompt_cache_breakpoints(
+            model=model, litellm_params=litellm_params
+        )
+        wire_safe_items: Final = tuple(
+            _item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input
+        )
+        if _same_objects(wire_safe_items, input):
+            return input
+        wire_safe_input: Final = list(wire_safe_items)
+        return cast("ResponseInputParam", wire_safe_input)  # cast-ok: items keep their shape, minus a rejected marker
+
+    def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool:
+        provider: Final = _provider_name(self.custom_llm_provider)
+        base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params)
+        known_support: Final = (
+            openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model))
+        )
+        return next((support for support in known_support if support is not None), None) is not False
+
     def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool:
         host: Final = _DeploymentHost.model_validate(litellm_params)
         return self.custom_llm_provider == LlmProviders.OPENAI and targets_openai_api(host.api_base or host.base_url)
@@ -688,14 +808,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
 
     @staticmethod
     def _without_foreign_tool_call_item_id(item: object) -> object:
-        if not isinstance(item, dict):
+        fields: Final = _json_object_or_none(item)
+        if fields is None:
             return item
-        item_type: Final = item.get("type")
-        item_id: Final = item.get("id")
+        item_type: Final = fields.get("type")
+        item_id: Final = fields.get("id")
         genuine_prefix: Final = TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE.get(item_type) if isinstance(item_type, str) else None
         if genuine_prefix is None or not isinstance(item_id, str) or item_id.startswith(genuine_prefix):
             return item
-        return {key: value for key, value in item.items() if key != "id"}
+        return {key: value for key, value in fields.items() if key != "id"}
 
     def _sanitized_tool_schemas_for_openai(
         self,
@@ -739,6 +860,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None
         return base_model if isinstance(base_model, str) and base_model else model
 
+    @staticmethod
+    def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str:
+        declared: Final = _DeclaredModel.model_validate(litellm_params)
+        model_info: Final = _json_object_or_none(declared.model_info)
+        candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model"))
+        return next((name for name in candidates if isinstance(name, str) and name), model)
+
     @staticmethod
     def _sanitized_tool_entry(
         entry: Mapping[str, object],
@@ -1094,7 +1222,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         """
         Transform the get response API request into a URL and data
 
@@ -1103,7 +1231,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         """
         encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
         url: Final = f"{api_base}/{encoded_response_id}"
-        data: Final[dict] = {}
+        data: Final[dict[str, str]] = {}
         return url, data
 
     def transform_get_response_api_response(
@@ -1218,7 +1346,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
     def compact_dropped_params(self, litellm_params: GenericLiteLLMParams) -> frozenset[str]:
         if self.custom_llm_provider not in _PROVIDERS_WITH_STRICT_COMPACT_PARAMS:
             return frozenset()
-        if _drops_params(litellm_params):
+        if drops_params(litellm_params):
             return _COMPACT_TRACKING_PARAMS | _COMPACT_REJECTED_PARAMS
         return _COMPACT_TRACKING_PARAMS
 

litellm/llms/openai_like/messages/transformation.py

@@ -37,6 +37,9 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
     def is_native_messages_route(self) -> bool:
         return False
 
+    def hoists_wrapped_defer_loading(self, model: str) -> bool:
+        return False
+
     def validate_anthropic_messages_environment(
         self,
         headers: dict[str, str],

litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py

@@ -1,4 +1,5 @@
 from collections.abc import Sequence
+from dataclasses import dataclass
 from typing import Final, Literal
 
 import httpx
@@ -8,7 +9,6 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.caching.caching import Cache, LiteLLMCacheType
-from litellm.constants import MINIMUM_PROMPT_CACHE_TOKEN_COUNT
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -16,10 +16,11 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
 )
 from litellm.llms.openai.openai import AllMessageValues
-from litellm.utils import is_prompt_caching_valid_prompt
+from litellm.types.llms.openai import ChatCompletionToolParam
+from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt
 
 from ..common_utils import VertexAIError, get_vertex_base_url
-from ..vertex_llm_base import VertexBase
+from ..vertex_llm_base import VertexBase, publisher_model_location
 from .known_cached_contents import KnownCachedContents, known_cached_contents
 from .transformation import (
     cached_messages_end_on_supported_turn,
@@ -65,6 +66,19 @@ class _CreatedCachedContentRoot(RootModel[_CreatedCachedContent]):
     model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
 
 
+_TOOL_PARAMS: Final = ("tools", "tool_choice")
+
+
+def _sync_cache_client(timeout: float | httpx.Timeout | None) -> HTTPHandler:
+    if timeout is None:
+        return HTTPHandler()
+    return HTTPHandler(timeout=httpx.Timeout(timeout) if isinstance(timeout, (float, int)) else timeout)
+
+
+class _CachedContentsNotServedError(VertexAIError):
+    pass
+
+
 def _record_cache_creation_tokens(logging_obj: Logging, created: _CreatedCachedContent) -> None:
     tokens: Final = (created.get("usageMetadata") or {}).get("totalTokenCount") or 0
     logging_obj.vertex_ai_cache_creation_input_tokens = max(tokens, 0)  # rebind-ok: set provider count
@@ -78,6 +92,61 @@ def _wait_seconds(timeout: float | httpx.Timeout | None) -> float | None:
             return timeout
 
 
+@dataclass(frozen=True, slots=True)
+class _CacheTarget:
+    model: str
+    project: str | None
+    location: str | None
+
+
+def _cache_target(
+    custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"],
+    api_base: str | None,
+    model: str,
+    vertex_project: str | None,
+    vertex_location: str | None,
+) -> _CacheTarget:
+    model_location: Final = (
+        None if custom_llm_provider == "gemini" or api_base is None else publisher_model_location(api_base)
+    )
+    if model_location is None:
+        return _CacheTarget(model=model, project=vertex_project, location=vertex_location)
+    return _CacheTarget(model=model_location.model, project=model_location.project, location=model_location.location)
+
+
+def _prompt_cache_min_tokens(models: Sequence[str], custom_llm_provider: str) -> int:
+    return max(
+        get_prompt_cache_min_tokens(
+            model if model.startswith(custom_llm_provider) else f"{custom_llm_provider}/{model}"
+        )
+        for model in models
+    )
+
+
+def _context_caching_skip_reason(
+    cached_messages: Sequence[AllMessageValues],
+    tools: Sequence[ChatCompletionToolParam] | None,
+    deployment_model: str,
+    cache_model: str,
+    custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"],
+) -> str | None:
+    if not cached_messages_end_on_supported_turn(cached_messages):
+        return (
+            "Vertex AI context caching: cached message block ends on a model turn once "
+            "system messages are extracted, which the cachedContents API rejects."
+        )
+    min_tokens: Final = _prompt_cache_min_tokens((deployment_model, cache_model), custom_llm_provider)
+    if not is_prompt_caching_valid_prompt(
+        model=cache_model,
+        messages=list(cached_messages),
+        tools=None if tools is None else list(tools),
+        custom_llm_provider=custom_llm_provider,
+        min_token_count=min_tokens,
+    ):
+        return f"Vertex AI context caching: cached content is below minimum token count ({min_tokens})."
+    return None
+
+
 class ContextCachingEndpoints(VertexBase):
     """
     Covers context caching endpoints for Vertex AI + Google AI Studio
@@ -135,6 +204,7 @@ class ContextCachingEndpoints(VertexBase):
             vertex_project=vertex_project,
             vertex_location=vertex_location,
             vertex_api_version=("v1beta1" if custom_llm_provider == "vertex_ai_beta" else "v1"),
+            collection_endpoint=True,
         )
 
     def check_cache(
@@ -206,6 +276,8 @@ class ContextCachingEndpoints(VertexBase):
             except httpx.HTTPStatusError as e:
                 if e.response.status_code == 403:
                     return None
+                if e.response.status_code == 404 and custom_llm_provider == "gemini":
+                    raise _CachedContentsNotServedError(status_code=404, message=e.response.text)
                 raise VertexAIError(status_code=e.response.status_code, message=e.response.text)
             except Exception as e:
                 raise VertexAIError(status_code=500, message=str(e))
@@ -305,6 +377,8 @@ class ContextCachingEndpoints(VertexBase):
             except httpx.HTTPStatusError as e:
                 if e.response.status_code == 403:
                     return None
+                if e.response.status_code == 404 and custom_llm_provider == "gemini":
+                    raise _CachedContentsNotServedError(status_code=404, message=e.response.text)
                 raise VertexAIError(status_code=e.response.status_code, message=e.response.text)
             except Exception as e:
                 raise VertexAIError(status_code=500, message=str(e))
@@ -370,41 +444,25 @@ class ContextCachingEndpoints(VertexBase):
         if len(cached_messages) == 0:
             return messages, optional_params, None
 
-        if not cached_messages_end_on_supported_turn(cached_messages):
-            verbose_logger.debug(
-                "Vertex AI context caching: cached message block ends on a model turn once "
-                "system messages are extracted, which the cachedContents API rejects. "
-                "Skipping context caching."
-            )
+        target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
+        tools: Final = optional_params.get("tools")
+        tool_choice: Final = optional_params.get("tool_choice")
+        skip_reason: Final = _context_caching_skip_reason(
+            cached_messages, tools, model, target.model, custom_llm_provider
+        )
+        if skip_reason is not None:
+            verbose_logger.debug("%s Skipping context caching.", skip_reason)
             return messages, optional_params, None
 
-        # Gemini requires a minimum of 1024 tokens for context caching.
-        # Skip caching if the cached content is too small to avoid API errors.
-        if not is_prompt_caching_valid_prompt(
-            model=model,
-            messages=cached_messages,
-            tools=optional_params.get("tools"),
-            custom_llm_provider=custom_llm_provider,
-        ):
-            verbose_logger.debug(
-                "Vertex AI context caching: cached content is below minimum token "
-                "count (%d). Skipping context caching.",
-                MINIMUM_PROMPT_CACHE_TOKEN_COUNT,
-            )
-            return messages, optional_params, None
-
-        tools: Final = optional_params.pop("tools", None)
-        tool_choice: Final = optional_params.pop("tool_choice", None)
-
         ## AUTHORIZATION ##
         token, url = self._get_token_and_url_context_caching(
             gemini_api_key=api_key,
             custom_llm_provider=custom_llm_provider,
             api_base=api_base,
-            vertex_project=vertex_project,
-            vertex_location=vertex_location,
+            vertex_project=target.project,
+            vertex_location=target.location,
             vertex_auth_header=vertex_auth_header,
-            model=model,
+            model=target.model,
         )
 
         headers: Final = {
@@ -417,44 +475,42 @@ class ContextCachingEndpoints(VertexBase):
         if extra_headers is not None:
             headers.update(extra_headers)
 
-        if client is None or not isinstance(client, HTTPHandler):
-            _params: Final = {}
-            if timeout is not None:
-                if isinstance(timeout, float) or isinstance(timeout, int):
-                    timeout = httpx.Timeout(timeout)
-                _params["timeout"] = timeout
-            client = HTTPHandler(**_params)
-        else:
-            client = client
+        http_client: Final = client if isinstance(client, HTTPHandler) else _sync_cache_client(timeout)
 
         ## CHECK IF CACHED ALREADY
         generated_cache_key: Final = local_cache_obj.get_cache_key(
-            messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model
-        )
-        google_cache_name: Final = self.check_cache(
-            cache_key=generated_cache_key,
-            client=client,
-            headers=headers,
-            api_key=api_key,
-            api_base=api_base,
-            logging_obj=logging_obj,
-            custom_llm_provider=custom_llm_provider,
-            vertex_project=vertex_project,
-            vertex_location=vertex_location,
-            vertex_auth_header=vertex_auth_header,
-            model=model,
+            messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model
         )
+        try:
+            google_cache_name: Final = self.check_cache(
+                cache_key=generated_cache_key,
+                client=http_client,
+                headers=headers,
+                api_key=api_key,
+                api_base=api_base,
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+                vertex_project=target.project,
+                vertex_location=target.location,
+                vertex_auth_header=vertex_auth_header,
+                model=target.model,
+            )
+        except _CachedContentsNotServedError:
+            verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url)
+            return messages, optional_params, None
+        for tool_param in _TOOL_PARAMS:
+            optional_params.pop(tool_param, None)
         if google_cache_name:
             return non_cached_messages, optional_params, google_cache_name
 
         ## TRANSFORM REQUEST
         cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching(
-            model=model,
+            model=target.model,
             messages=cached_messages,
             cache_key=generated_cache_key,
             custom_llm_provider=custom_llm_provider,
-            vertex_project=vertex_project,
-            vertex_location=vertex_location,
+            vertex_project=target.project,
+            vertex_location=target.location,
         )
 
         cached_content_request_body["tools"] = tools
@@ -473,7 +529,7 @@ class ContextCachingEndpoints(VertexBase):
         )
 
         try:
-            response: Final = client.post(
+            response: Final = http_client.post(
                 url=url,
                 headers=headers,
                 json=cached_content_request_body,
@@ -489,7 +545,7 @@ class ContextCachingEndpoints(VertexBase):
         _record_cache_creation_tokens(logging_obj, raw_response_cached)
         self._known_caches.remember(
             KnownCachedContents.scope(
-                url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location
+                url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location
             ),
             generated_cache_key,
             raw_response_cached.get("name"),
@@ -536,41 +592,25 @@ class ContextCachingEndpoints(VertexBase):
         if len(cached_messages) == 0:
             return messages, optional_params, None
 
-        if not cached_messages_end_on_supported_turn(cached_messages):
-            verbose_logger.debug(
-                "Vertex AI context caching: cached message block ends on a model turn once "
-                "system messages are extracted, which the cachedContents API rejects. "
-                "Skipping context caching."
-            )
+        target: Final = _cache_target(custom_llm_provider, api_base, model, vertex_project, vertex_location)
+        tools: Final = optional_params.get("tools")
+        tool_choice: Final = optional_params.get("tool_choice")
+        skip_reason: Final = _context_caching_skip_reason(
+            cached_messages, tools, model, target.model, custom_llm_provider
+        )
+        if skip_reason is not None:
+            verbose_logger.debug("%s Skipping context caching.", skip_reason)
             return messages, optional_params, None
 
-        # Gemini requires a minimum of 1024 tokens for context caching.
-        # Skip caching if the cached content is too small to avoid API errors.
-        if not is_prompt_caching_valid_prompt(
-            model=model,
-            messages=cached_messages,
-            tools=optional_params.get("tools"),
-            custom_llm_provider=custom_llm_provider,
-        ):
-            verbose_logger.debug(
-                "Vertex AI context caching: cached content is below minimum token "
-                "count (%d). Skipping context caching.",
-                MINIMUM_PROMPT_CACHE_TOKEN_COUNT,
-            )
-            return messages, optional_params, None
-
-        tools: Final = optional_params.pop("tools", None)
-        tool_choice: Final = optional_params.pop("tool_choice", None)
-
         ## AUTHORIZATION ##
         token, url = self._get_token_and_url_context_caching(
             gemini_api_key=api_key,
             custom_llm_provider=custom_llm_provider,
             api_base=api_base,
-            vertex_project=vertex_project,
-            vertex_location=vertex_location,
+            vertex_project=target.project,
+            vertex_location=target.location,
             vertex_auth_header=vertex_auth_header,
-            model=model,
+            model=target.model,
         )
 
         headers: Final = {
@@ -590,10 +630,10 @@ class ContextCachingEndpoints(VertexBase):
 
         ## CHECK IF CACHED ALREADY
         generated_cache_key: Final = local_cache_obj.get_cache_key(
-            messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model
+            messages=cached_messages, tools=tools, tool_choice=tool_choice, model=target.model
         )
         scope: Final = KnownCachedContents.scope(
-            url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location
+            url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location
         )
 
         async def lookup_or_create() -> str | None:
@@ -605,10 +645,10 @@ class ContextCachingEndpoints(VertexBase):
                 api_base=api_base,
                 logging_obj=logging_obj,
                 custom_llm_provider=custom_llm_provider,
-                vertex_project=vertex_project,
-                vertex_location=vertex_location,
+                vertex_project=target.project,
+                vertex_location=target.location,
                 vertex_auth_header=vertex_auth_header,
-                model=model,
+                model=target.model,
             )
 
             if google_cache_name:
@@ -616,12 +656,12 @@ class ContextCachingEndpoints(VertexBase):
 
             ## TRANSFORM REQUEST
             cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching(
-                model=model,
+                model=target.model,
                 messages=cached_messages,
                 cache_key=generated_cache_key,
                 custom_llm_provider=custom_llm_provider,
-                vertex_project=vertex_project,
-                vertex_location=vertex_location,
+                vertex_project=target.project,
+                vertex_location=target.location,
             )
 
             cached_content_request_body["tools"] = tools
@@ -659,9 +699,15 @@ class ContextCachingEndpoints(VertexBase):
             )
             return raw_response_cached.get("name")
 
-        cache_name: Final = await self._known_caches.single_flight(
-            scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout)
-        )
+        try:
+            cache_name: Final = await self._known_caches.single_flight(
+                scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout)
+            )
+        except _CachedContentsNotServedError:
+            verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url)
+            return messages, optional_params, None
+        for tool_param in _TOOL_PARAMS:
+            optional_params.pop(tool_param, None)
         return non_cached_messages, optional_params, cache_name
 
     def get_cache(self):

litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py

@@ -6,9 +6,12 @@ Unlike Gemini models which use Google's token counting API, partner models use
 their respective publisher-specific count-tokens endpoints.
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@@ -20,6 +23,22 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
 )
 
 
+def count_tokens_request_body(
+    request_data: Mapping[str, object], model: str, drop_params: object
+) -> Mapping[str, object]:
+    tools: Final = request_data.get("tools")
+    if not isinstance(tools, list):
+        return request_data
+    return {
+        **request_data,
+        "tools": tools_without_custom_field(
+            OBJECT_LIST_ADAPTER.validate_python(tools),
+            hoist_defer_loading=is_claude_model_target("vertex_ai", model),
+            drop_custom=drops_unsupported_params(drop_params),
+        ),
+    }
+
+
 def count_tokens_publisher(model: str) -> str | None:
     return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None)
 
@@ -169,7 +188,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
         response: Final = await async_client.post(
             endpoint_url,
             headers=headers,
-            json=request_data,
+            json=dict(count_tokens_request_body(request_data, model, litellm_params.get("drop_params"))),
             timeout=30.0,
         )
 

litellm/llms/vertex_ai/vertex_llm_base.py

@@ -6,9 +6,11 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S
 
 import asyncio
 import json
+import re
 import threading
 from collections import OrderedDict
 from collections.abc import Mapping
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
 from urllib.parse import urlparse
 
@@ -46,6 +48,31 @@ def graft_default_vertex_path(api_base: str, default_url: str) -> str:
 
 _graft_default_vertex_path = graft_default_vertex_path
 
+
+_PUBLISHER_MODEL_LOCATION: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+(?=/publishers/[^/]+/models/[^/]+/?$)")
+
+
+@dataclass(frozen=True, slots=True)
+class PublisherModelLocation:
+    parent_url: str
+    project: str
+    location: str
+    model: str
+
+
+def publisher_model_location(api_base: str) -> PublisherModelLocation | None:
+    match: Final = _PUBLISHER_MODEL_LOCATION.match(api_base)
+    if match is None:
+        return None
+    _, _, project, _, location = match.group().rsplit("/", 4)
+    return PublisherModelLocation(
+        parent_url=match.group(),
+        project=project,
+        location=location,
+        model=api_base.rstrip("/").rsplit("/", 1)[-1],
+    )
+
+
 GOOGLE_IMPORT_ERROR_MESSAGE: Final = (
     "Google Cloud SDK not found. Install it with: pip install 'litellm[google]' or pip install google-cloud-aiplatform"
 )
@@ -637,15 +664,18 @@ class VertexBase:
         vertex_location: str | None = None,
         vertex_api_version: Literal["v1", "v1beta1"] | None = None,
         use_psc_endpoint_format: bool = False,
+        collection_endpoint: bool = False,
     ) -> tuple[str | None, str]:
         """
         for cloudflare ai gateway - https://github.com/BerriAI/litellm/issues/4317
 
         Handles custom api_base for:
-        1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}
+        1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}, or {api_base}/{endpoint}
+           for a collection endpoint
         2. Vertex AI with standard proxies - grafts the default vertex URL path onto the
            api_base when its path is empty or only an API version (/v1, /v1beta1);
-           otherwise constructs {api_base}:{endpoint}
+           otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint under an api_base
+           naming a publisher model, the collection under that model's projects/{project}/locations/{location}
         3. Vertex AI with PSC endpoints - constructs full path structure
            {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}
            (only when use_psc_endpoint_format=True)
@@ -653,22 +683,22 @@ class VertexBase:
         Args:
             use_psc_endpoint_format: If True, constructs PSC endpoint URL format.
                                      If False (default), uses api_base as-is and appends :{endpoint}
+            collection_endpoint: True when `endpoint` names a collection resource such as
+                                 cachedContents rather than a model action such as generateContent
 
         ## Returns
         - (auth_header, url) - Tuple[Optional[str], str]
         """
         if api_base:
             if custom_llm_provider == "gemini":
-                # For Gemini (Google AI Studio), construct the full path like other providers
-                if model is None:
+                if model is None and not collection_endpoint:
                     raise ValueError("Model parameter is required for Gemini custom API base URLs")
-                url = join_api_base(api_base, f"models/{model}:{endpoint}")
+                url = join_api_base(api_base, endpoint if collection_endpoint else f"models/{model}:{endpoint}")
                 if gemini_api_key is None:
                     raise ValueError(
                         "Missing Gemini API key. Set the GEMINI_API_KEY or GOOGLE_API_KEY environment variable."
                     )
-                if gemini_api_key is not None:
-                    auth_header = {"x-goog-api-key": gemini_api_key}
+                auth_header = {"x-goog-api-key": gemini_api_key}
             else:
                 # For Vertex AI
                 if use_psc_endpoint_format:
@@ -692,7 +722,12 @@ class VertexBase:
                 elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path:
                     url = graft_default_vertex_path(api_base=api_base, default_url=url)
                 else:
-                    url = f"{api_base.rstrip('/')}:{endpoint}"
+                    model_location: Final = publisher_model_location(api_base) if collection_endpoint else None
+                    url = (
+                        f"{api_base.rstrip('/')}:{endpoint}"
+                        if model_location is None
+                        else join_api_base(model_location.parent_url, endpoint)
+                    )
             if stream is True:
                 parsed_stream_url: Final = urlparse(url)
                 stream_query: Final = f"{parsed_stream_url.query}&alt=sse" if parsed_stream_url.query else "alt=sse"

litellm/llms/volcengine/responses/transformation.py

@@ -313,10 +313,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
         url: Final = f"{api_base}/{encoded_response_id}"
-        data: Final[dict] = {}
+        data: Final[dict[str, str]] = {}
         return url, data
 
     def transform_get_response_api_response(

litellm/main.py

@@ -234,6 +234,7 @@ from .litellm_core_utils.streaming_chunk_builder_utils import (
     ChunkProcessor as ChunkProcessor,  # noqa: PLC0414  # public re-export
 )
 from .llms.anthropic.chat import AnthropicChatCompletion
+from .llms.anthropic.common_utils import is_claude_tool_target
 from .llms.azure.audio_transcriptions import AzureAudioTranscription
 from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params
 from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion
@@ -1135,17 +1136,6 @@ from litellm.responses.bridge_check import (  # noqa: E402  # re-export at the o
 _ANTHROPIC_ONLY_TOOL_KEYS: Final = frozenset({"input_examples", "eager_input_streaming"})
 
 
-def _is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool:
-    if custom_llm_provider == "anthropic":
-        return True
-    model_lower: Final = model.lower()
-    if custom_llm_provider == "bedrock":
-        return "claude" in model_lower or ("arn:" in model_lower and ":bedrock:" in model_lower)
-    if custom_llm_provider == "azure_ai" or custom_llm_provider == "vertex_ai":
-        return "claude" in model_lower
-    return False
-
-
 def _history_without_unanswered_tool_search_calls(
     messages: Sequence[object], custom_llm_provider: str | None, provider_config: BaseConfig
 ) -> tuple[object, ...] | None:
@@ -2672,10 +2662,13 @@ def _complete_custom_openai(
 
     # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
     api_base = _resolve_openai_api_base(api_base)
+    if custom_llm_provider == "bigmodel" and not api_key:
+        raise litellm.AuthenticationError(
+            message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
+        )
     organization = (
         organization
-        or litellm.organization
-        or get_secret("OPENAI_ORGANIZATION")
+        or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION"))
         or None  # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
     )
     openai.organization = organization
@@ -5749,7 +5742,7 @@ def completion(
             }
         )
 
-        if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model):
+        if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model):
             tools = _drop_anthropic_only_tool_keys(tools=tools)
 
         if provider_specific_header is not None:
@@ -6299,6 +6292,7 @@ def completion(
             or custom_llm_provider == "nebius"
             or custom_llm_provider == "wandb"
             or custom_llm_provider == "clarifai"
+            or custom_llm_provider == "bigmodel"
             or custom_llm_provider in litellm.openai_compatible_providers
             or JSONProviderRegistry.exists(custom_llm_provider)  # JSON-configured providers
             or "ft:gpt-3.5-turbo" in model  # finetune gpt-3.5-turbo

litellm/messages/dispatch.py

@@ -111,6 +111,7 @@ _FEATURES: Final[detectors.Detectors] = MappingProxyType(
         Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection,
         Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection,
         Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry,
+        Feature.TOOL_CUSTOM_FIELD: route_detectors.tool_custom_field,
     }
 )
 

litellm/model_prices_and_context_window_backup.json

@@ -28071,7 +28071,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": true
     },
     "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
         "input_cost_per_token": 1.2e-06,
@@ -28345,7 +28345,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": true
     },
     "fireworks_ai/qwen3p7-plus": {
         "cache_read_input_token_cost": 8e-08,
@@ -80243,5 +80243,81 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true
+    },
+    "bigmodel/glm-5.3": {
+        "input_cost_per_token": 1.1428571428571428e-06,
+        "output_cost_per_token": 4e-06,
+        "cache_read_input_token_cost": 2.857142857142857e-07,
+        "litellm_provider": "bigmodel",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "thinking_always_on": true,
+        "metadata": {
+            "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
+        },
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_native_streaming": true
+    },
+    "bigmodel/glm-5.3-flash": {
+        "input_cost_per_token": 1.142857142857143e-07,
+        "output_cost_per_token": 4e-07,
+        "cache_read_input_token_cost": 3.2857142857142855e-08,
+        "litellm_provider": "bigmodel",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "thinking_always_on": true,
+        "supports_vision": true,
+        "supports_video_input": true,
+        "metadata": {
+            "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
+        },
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_native_streaming": true
+    },
+    "bigmodel/glm-5.3-flashx": {
+        "input_cost_per_token": 2.857142857142857e-07,
+        "output_cost_per_token": 1e-06,
+        "cache_read_input_token_cost": 8.142857142857142e-08,
+        "litellm_provider": "bigmodel",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "thinking_always_on": true,
+        "supports_vision": true,
+        "supports_video_input": true,
+        "metadata": {
+            "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
+        },
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_native_streaming": true
     }
 }

litellm/provider_endpoints_support_backup.json

@@ -5352,6 +5352,32 @@
         "a2a": true,
         "generateContent": true
       }
+    },
+    "bigmodel": {
+      "display_name": "BigModel (Zhipu AI) (`bigmodel`)",
+      "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction",
+      "endpoints": {
+        "chat_completions": true,
+        "responses": true,
+        "text_completion": true,
+        "embeddings": false,
+        "image_generations": false,
+        "image_edits": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "rerank": false,
+        "ocr": false,
+        "search": false,
+        "moderations": false,
+        "realtime": false,
+        "video_generations": false,
+        "batches": false,
+        "messages": true,
+        "interactions": true,
+        "systemone": false,
+        "a2a": true,
+        "generateContent": true
+      }
     }
   },
   "endpoints": {

litellm/proxy/_lazy_openapi_snapshot.json

@@ -7816,7 +7816,7 @@
     "paths": {
       "/claude-code/marketplace.json": {
         "get": {
-          "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add <url>\n- claude plugin install <name>@<marketplace>\n\nWithout `key` the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` or an access group on the key or its team.\n\nReturns:\n    Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n    ```bash\n    claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n    claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n    claude plugin install my-plugin@litellm\n    ```",
+          "description": "Serve marketplace.json for Claude Code plugin discovery.\n\nThis endpoint is accessed by Claude Code CLI when users run:\n- claude plugin marketplace add <url>\n- claude plugin install <name>@<marketplace>\n\nWithout a key the catalog holds the enabled (public) plugins. With `?key=sk-...`\nthe key is authenticated and the catalog also holds the disabled plugins granted\nto it through `object_permission.skills` or an access group on the key or its team.\n\nWith `general_settings.claude_code_marketplace_auth_required: true` the key must come\nin the `Authorization` or `x-litellm-api-key` header (Claude Code sends them from the\nmarketplace source's `headers`), `?key=` is ignored, and a caller with no valid key\ngets a 401 with `WWW-Authenticate: Bearer`.\n\nReturns:\n    Marketplace catalog with list of available plugins and their git sources.\n\nExample:\n    ```bash\n    claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json\n    claude plugin marketplace add \"http://localhost:4000/claude-code/marketplace.json?key=sk-...\"\n    claude plugin install my-plugin@litellm\n    ```",
           "operationId": "get_marketplace_claude_code_marketplace_json_get",
           "parameters": [
             {
@@ -33639,7 +33639,7 @@
           ]
         },
         "get": {
-          "description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123     -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123     -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```",
+          "description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123     -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123     -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Resume a background response's event stream after sequence number 7\ncurl -N \"http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7\"     -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```",
           "operationId": "get_response_openai_v1_responses__response_id__get",
           "parameters": [
             {
@@ -33650,6 +33650,46 @@
                 "title": "Response Id",
                 "type": "string"
               }
+            },
+            {
+              "description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)",
+              "in": "query",
+              "name": "stream",
+              "required": false,
+              "schema": {
+                "type": "boolean"
+              }
+            },
+            {
+              "description": "With stream=true, replay only the events after this sequence number",
+              "in": "query",
+              "name": "starting_after",
+              "required": false,
+              "schema": {
+                "minimum": 0,
+                "type": "integer"
+              }
+            },
+            {
+              "description": "Additional output data to include, forwarded to the provider",
+              "in": "query",
+              "name": "include",
+              "required": false,
+              "schema": {
+                "items": {
+                  "type": "string"
+                },
+                "type": "array"
+              }
+            },
+            {
+              "description": "With stream=true, whether delta events carry obfuscation padding",
+              "in": "query",
+              "name": "include_obfuscation",
+              "required": false,
+              "schema": {
+                "type": "boolean"
+              }
             }
           ],
           "responses": {
@@ -33659,6 +33699,12 @@
                   "schema": {
                     "$ref": "#/components/schemas/ResponsesAPIResponse"
                   }
+                },
+                "text/event-stream": {
+                  "schema": {
+                    "description": "Server sent events when stream=true",
+                    "type": "string"
+                  }
                 }
               },
               "description": "Successful Response"

litellm/proxy/_types.py

@@ -2833,6 +2833,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
         None,
         description="If True, stores request messages and responses in spend logs. Default is False.",
     )
+    claude_code_marketplace_auth_required: bool = Field(
+        False,
+        description="If True, GET /claude-code/marketplace.json requires a LiteLLM key in the Authorization or x-litellm-api-key header (?key= is not accepted) and answers 401 with WWW-Authenticate: Bearer without one, and GET /public/skill_hub returns an empty list. Default is False, which serves the enabled plugins to anyone.",
+    )
     disable_requester_ip_address_logging: bool | None = Field(
         None,
         description="If True, new spend log rows store requester_ip_address as null in the column, the metadata and the stored request. The stored request also drops client-IP forwarding headers such as x-forwarded-for, and IP addresses in its other header values and in stored error messages are masked. Guardrails, metrics and callbacks that read the standard logging payload still receive the IP. Default is False.",

litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py

@@ -7,7 +7,8 @@ Actual plugin files are hosted on GitHub/GitLab/Bitbucket or as a zip archive on
 any HTTPS host (S3, Artifactory, a static file server).
 
 Endpoints:
-/claude-code/marketplace.json  - GET  - List plugins for Claude Code discovery (unauthenticated; `?key=` adds the key's granted skills)
+/claude-code/marketplace.json  - GET  - List plugins for Claude Code discovery (anonymous unless
+                                 general_settings.claude_code_marketplace_auth_required; a key adds its granted skills)
 /claude-code/plugins           - POST - Register a new plugin (create-only, proxy admin only)
 /claude-code/plugins           - GET  - List plugins visible to the key (enabled, plus granted disabled ones)
 /claude-code/plugins/{name}    - GET  - Get plugin details (403 on a disabled plugin the key is not granted)
@@ -31,14 +32,17 @@ from pydantic import TypeAdapter
 from typing_extensions import ReadOnly
 
 from litellm._logging import verbose_proxy_logger
-from litellm.proxy._types import CommonProxyErrors, ProxyException, UserAPIKeyAuth
+from litellm.proxy._types import CommonProxyErrors, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
 from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import (
     SkillVisibility,
     resolve_skill_visibility,
 )
+from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required
 from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
 from litellm.repositories.table_repositories import ClaudeCodePluginRepository
+from litellm.types.proxy.auth.special_headers import SpecialHeaders
 from litellm.types.proxy.claude_code_endpoints import (
     InstallationPreference,
     ListPluginsResponse,
@@ -109,6 +113,41 @@ async def get_prisma_client() -> object:
 _get_prisma_client: Final = get_prisma_client
 
 
+async def _header_key_caller(request: Request) -> UserAPIKeyAuth:
+    try:
+        caller: Final = await user_api_key_auth(
+            request=request,
+            api_key=request.headers.get(SpecialHeaders.openai_authorization.value) or "",
+            custom_litellm_key_header=request.headers.get(SpecialHeaders.custom_litellm_api_key.value),
+        )
+    except ProxyException as e:
+        if e.code != "401":
+            raise
+        raise ProxyException(
+            message=e.message,
+            type=e.type,
+            param=e.param,
+            code=401,
+            headers={**e.headers, "WWW-Authenticate": "Bearer"},
+        ) from e
+    if caller.user_id == DB_UNAVAILABLE_FALLBACK_USER_ID:
+        raise ProxyException(
+            message="The LiteLLM key could not be verified because the database is unavailable",
+            type=ProxyErrorTypes.no_db_connection,
+            param=None,
+            code=503,
+        )
+    return caller
+
+
+async def _marketplace_caller(request: Request, key: str | None) -> UserAPIKeyAuth | None:
+    if marketplace_auth_required():
+        return await _header_key_caller(request)
+    if key:
+        return await user_api_key_auth(request=request, api_key=f"Bearer {key}")
+    return None
+
+
 @router.get(
     "/claude-code/marketplace.json",
     tags=["Claude Code Marketplace"],
@@ -121,10 +160,15 @@ async def get_marketplace(request: Request, key: str | None = None):
     - claude plugin marketplace add <url>
     - claude plugin install <name>@<marketplace>
 
-    Without `key` the catalog holds the enabled (public) plugins. With `?key=sk-...`
+    Without a key the catalog holds the enabled (public) plugins. With `?key=sk-...`
     the key is authenticated and the catalog also holds the disabled plugins granted
     to it through `object_permission.skills` or an access group on the key or its team.
 
+    With `general_settings.claude_code_marketplace_auth_required: true` the key must come
+    in the `Authorization` or `x-litellm-api-key` header (Claude Code sends them from the
+    marketplace source's `headers`), `?key=` is ignored, and a caller with no valid key
+    gets a 401 with `WWW-Authenticate: Bearer`.
+
     Returns:
         Marketplace catalog with list of available plugins and their git sources.
 
@@ -137,10 +181,7 @@ async def get_marketplace(request: Request, key: str | None = None):
     """
     try:
         prisma_client: Final = await get_prisma_client()
-
-        caller: Final[UserAPIKeyAuth | None] = (
-            await user_api_key_auth(request=request, api_key=f"Bearer {key}") if key else None
-        )
+        caller: Final = await _marketplace_caller(request, key)
         visibility: Final[SkillVisibility] = await resolve_skill_visibility(caller)
         plugins: Final[Sequence[_PluginRecord]] = await ClaudeCodePluginRepository(prisma_client).table.find_many(
             where=visibility.where()
@@ -201,7 +242,7 @@ async def get_marketplace(request: Request, key: str | None = None):
         verbose_proxy_logger.exception("Error generating marketplace: %s", e)
         raise HTTPException(
             status_code=500,
-            detail={"error": f"Failed to generate marketplace: {e}"},
+            detail={"error": "Failed to generate marketplace"},
         )
 
 

litellm/proxy/anthropic_endpoints/gateway_endpoints.py

@@ -45,7 +45,11 @@ from litellm.proxy.common_utils.http_parsing_utils import (  # noqa: F401  # leg
     safe_set_request_parsed_body,
 )
 from litellm.proxy.management_endpoints.sso_helper_utils import CLI_SSO_SESSIONS_TARGET
-from litellm.proxy.management_endpoints.ui_sso import CliSsoTeamDetail
+from litellm.proxy.management_endpoints.ui_sso import (
+    CLI_SSO_BROWSER_TEAM_CHOICE,
+    CLI_SSO_CHOSEN_TEAM_ID,
+    CliSsoTeamDetail,
+)
 from litellm.proxy.mcp_server.gateway_dcr_flow import (
     PROXY_API_AUDIENCE,
     MintedProxyCredential,
@@ -235,6 +239,7 @@ async def device_authorization(request: Request) -> JSONResponse:
         "sso_complete": False,
         "user_code_verified": False,
         "session_data": None,
+        CLI_SSO_BROWSER_TEAM_CHOICE: True,
     }
     _set_cli_sso_flow(login_id=login_id, cache=cli_sso_session_cache, flow=flow)
 
@@ -267,7 +272,15 @@ def _validate_login(flow: Mapping[str, object]) -> _GatewayLogin | _OAuthError:
             status_code=400, error="invalid_grant", description="The login session is malformed; sign in again"
         )
 
-    team_id: Final = session_data.teams[0] if session_data.teams else None
+    chosen: Final = flow.get(CLI_SSO_CHOSEN_TEAM_ID)
+    chosen_team_id: Final = chosen if isinstance(chosen, str) else None
+    if chosen_team_id is not None and chosen_team_id not in session_data.teams:
+        return _OAuthError(
+            status_code=400,
+            error="invalid_grant",
+            description="The team picked at sign-in is no longer one of your teams; sign in again",
+        )
+    team_id: Final = chosen_team_id or next(iter(session_data.teams), None)
     selected_team: Final = selected_cli_sso_team_detail(team_details=session_data.team_details, team_id=team_id)
     if selected_team is None:
         return _OAuthError(

litellm/proxy/auth/auth_checks.py

@@ -46,6 +46,11 @@ from litellm.constants import (
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
+from litellm.litellm_core_utils.tool_fields import DropPolicy
+from litellm.llms.base_llm.managed_resources.utils import (
+    extract_provider_resource_id_from_unified_id,
+    is_base64_encoded_unified_id,
+)
 from litellm.models.project import LiteLLM_ProjectTable
 from litellm.proxy._types import (
     MODEL_LIST_ROUTES,
@@ -165,6 +170,7 @@ from litellm.types.proxy.auth.auth_checks import UserNotFoundError
 from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
 from litellm.types.router import DeploymentTypedDict
 from litellm.utils import get_utc_datetime
+from litellm.vector_stores.vector_store_registry import listed_vector_store_ids, vector_store_ids_in_tool
 
 from .auth_checks_organization import (
     add_team_org_context_to_request_body,
@@ -1514,6 +1520,7 @@ async def common_checks(
             valid_token=valid_token,
             user_object=user_object,
             deny_by_default=_vector_store_deny_by_default(_typed_request_body(general_settings)),
+            llm_router=llm_router,
         )
 
     # 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path)
@@ -7985,6 +7992,57 @@ def _strict_requested_vector_store_ids(request_body: Mapping[str, object]) -> tu
     )
 
 
+@dataclass(frozen=True, slots=True)
+class _UnreadableVectorStoreIds:
+    pass
+
+
+def _top_level_vector_store_ids(
+    vector_store_ids: object, drop_policies: Callable[[], tuple[DropPolicy, ...]]
+) -> tuple[str, ...] | _UnreadableVectorStoreIds:
+    listed: Final = listed_vector_store_ids(vector_store_ids)
+    if vector_store_ids is None or vector_store_ids == [*listed]:
+        return listed
+    if any(policy.drops("vector_store_ids") for policy in drop_policies()):
+        return listed
+    if isinstance(vector_store_ids, str):
+        return (vector_store_ids,)
+    return _UnreadableVectorStoreIds()
+
+
+def _legacy_requested_vector_store_ids(
+    request_body: Mapping[str, object], drop_policies: Callable[[], tuple[DropPolicy, ...]]
+) -> tuple[str, ...] | _UnreadableVectorStoreIds:
+    """
+    Reads tools the way VectorStoreRegistry does and leaves a malformed tool to the proxy's tool
+    validation. A top-level vector_store_ids string names one store, since a query parameter always
+    arrives as a string. Any other value that is not a list of strings names no readable stores unless a
+    drop policy the request can resolve to drops it, and then only the string entries of a list name
+    stores, as in registry retrieval.
+    """
+    top_level: Final = _top_level_vector_store_ids(request_body.get("vector_store_ids"), drop_policies)
+    if isinstance(top_level, _UnreadableVectorStoreIds):
+        return top_level
+    return tuple(
+        chain(
+            top_level,
+            chain.from_iterable(vector_store_ids_in_tool(tool) for tool in _tools(request_body.get("tools"))),
+        )
+    )
+
+
+def _restricts_vector_stores(grants: Sequence[tuple[GrantLayer, LiteLLM_ObjectPermissionTable | None]]) -> bool:
+    return any(grant is not None and bool(grant.vector_stores) for _, grant in grants)
+
+
+def _request_drop_policies(
+    request_body: Mapping[str, object], llm_router: Router | None, valid_token: UserAPIKeyAuth | None
+) -> tuple[DropPolicy, ...]:
+    from litellm.proxy.common_request_processing import counted_drop_policies
+
+    return counted_drop_policies(request_body, llm_router, valid_token.team_id if valid_token is not None else None)
+
+
 def _require_vector_store_grant(
     object_type: GrantLayer,
     vector_store_ids_to_run: Sequence[str],
@@ -8032,6 +8090,7 @@ async def vector_store_access_check(
     *,
     user_object: LiteLLM_UserTable | None = None,
     deny_by_default: bool = False,
+    llm_router: Router | None = None,
 ):
     """
     Checks whether the caller may use every vector store the request names.
@@ -8040,11 +8099,19 @@ async def vector_store_access_check(
     `retrieval_config.vector_store_id`. Grants come from each identity's
     `object_permission.vector_stores`.
 
-    With `deny_by_default=False` (legacy), a key or team only restricts access when its list is
-    nonempty, and stores are only read from the request when a vector store registry is loaded.
+    Both modes read the same request fields, with or without a vector store registry, and answer 400
+    when the top-level `vector_store_ids` is not a list of strings, which legacy mode does only when the
+    key or team has an allowlist and the value is not a string, which it checks as one store id. A
+    LiteLLM-managed store id is compared by the provider store id it decodes to, on both sides. Legacy mode skips that value instead when the request, `litellm_settings`, the router defaults or a deployment of the requested
+    model drops it (`drop_params`, or `additional_drop_params` naming it), the same settings the proxy's
+    tool validation reads, and reads tools the way the registry does, where strict mode answers 400 on a
+    malformed one.
 
-    With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), stores are read
-    even without a registry, and a missing record, `null` or `[]` grants nothing:
+    With `deny_by_default=False` (legacy), a key or team only restricts access when its list is
+    nonempty.
+
+    With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), a missing record,
+    `null` or `[]` grants nothing:
 
     - virtual key: the key must grant every store, and so must its team when it has one, even if
       the team failed to load
@@ -8056,8 +8123,8 @@ async def vector_store_access_check(
     rescue nor restrict a key or team request.
 
     Raises ProxyException (401, `{key,team,user}_vector_store_access_denied`) on the first identity
-    that does not grant a requested store, and with the flag on, ProxyException (400,
-    `invalid_request_error`) when a `vector_store_ids` field is not a list of strings.
+    that does not grant a requested store, and ProxyException (400, `invalid_request_error`) on a
+    malformed `vector_store_ids` the request does not drop and, in legacy mode, an allowlist applies to.
     """
     from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
 
@@ -8068,21 +8135,23 @@ async def vector_store_access_check(
         verbose_proxy_logger.debug("Prisma client not found, skipping vector store access check")
         return True
 
-    registry_ids: Final = (
-        _strict_requested_vector_store_ids(_typed_request_body(request_body))
+    typed_request_body: Final = _typed_request_body(request_body)
+    requested_ids: Final = (
+        _strict_requested_vector_store_ids(typed_request_body)
         if deny_by_default
-        else (
-            litellm.vector_store_registry.get_vector_store_ids_to_run(
-                non_default_params=request_body, tools=request_body.get("tools", None)
-            )
-            if litellm.vector_store_registry is not None
-            else None
+        else _legacy_requested_vector_store_ids(
+            typed_request_body, lambda: _request_drop_policies(typed_request_body, llm_router, valid_token)
         )
-        or ()
     )
-    rag_vector_store_id: Final = _get_rag_query_vector_store_id(_typed_request_body(request_body))
+    if isinstance(requested_ids, _UnreadableVectorStoreIds):
+        if _restricts_vector_stores(
+            await _identity_grants(valid_token, team_object, None, prisma_client, user_api_key_cache)
+        ):
+            raise _malformed_vector_store_ids()
+        return True
+    rag_vector_store_id: Final = _get_rag_query_vector_store_id(typed_request_body)
     rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else ()
-    vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids)))
+    vector_store_ids_to_run: Final = tuple(dict.fromkeys((*requested_ids, *rag_ids)))
     if not vector_store_ids_to_run:
         return True
 
@@ -8109,6 +8178,12 @@ async def vector_store_access_check(
     return True
 
 
+def _provider_vector_store_id(vector_store_id: str) -> str:
+    if not is_base64_encoded_unified_id(vector_store_id):
+        return vector_store_id
+    return extract_provider_resource_id_from_unified_id(vector_store_id) or vector_store_id
+
+
 def _can_object_call_vector_stores(
     object_type: Literal["key", "team", "org", "user"],
     vector_store_ids_to_run: Sequence[str],
@@ -8127,8 +8202,9 @@ def _can_object_call_vector_stores(
     if len(object_permissions.vector_stores) == 0:
         return True
 
+    allowed_stores: Final = frozenset(map(_provider_vector_store_id, object_permissions.vector_stores))
     for vector_store_id in vector_store_ids_to_run:
-        if vector_store_id not in object_permissions.vector_stores:
+        if _provider_vector_store_id(vector_store_id) not in allowed_stores:
             raise ProxyException(
                 message=f"User not allowed to access vector store. Tried to access {vector_store_id}. Only allowed to access {object_permissions.vector_stores}",
                 type=ProxyErrorTypes.get_vector_store_access_error_type_for_object(object_type),

litellm/proxy/auth/model_checks.py

@@ -20,6 +20,8 @@ from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
 from litellm.types.utils import LlmProviders
 from litellm.utils import ProviderConfigManager, get_valid_models
 
+LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
+
 _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
 
 
@@ -458,23 +460,21 @@ def get_known_models_from_wildcard(
             # add model prefix to wildcard models
             wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models]
 
-    known_providers: Final = {provider.value for provider in LlmProviders}
-    suffix_appended_wildcard_models: Final = []
-    for model in wildcard_models:
-        if not model.startswith(f"{wildcard_provider_prefix}/"):
-            # `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b").
-            # When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish
-            # multiple instances), replace that existing provider prefix instead of stacking
-            # both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b".
-            # Only strip the leading segment when it is a known provider, so ids whose first
-            # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it.
-            leading, sep, model_suffix = model.partition("/")
-            if sep and leading in known_providers:
-                model = f"{wildcard_provider_prefix}/{model_suffix}"
-            else:
-                model = f"{wildcard_provider_prefix}/{model}"
-        suffix_appended_wildcard_models.append(model)
-    return list(dict.fromkeys(suffix_appended_wildcard_models))
+    return list(
+        dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models)
+    )
+
+
+def _under_wildcard_prefix(model: str, wildcard_provider_prefix: str, listed_provider: str) -> str:
+    if model.startswith(f"{wildcard_provider_prefix}/"):
+        return model
+    leading, sep, model_suffix = model.partition("/")
+    leading_is_listed_providers_own_prefix: Final = (
+        sep != ""
+        and leading in LLM_PROVIDER_PREFIXES
+        and (leading == listed_provider or listed_provider.startswith(f"{leading}_"))
+    )
+    return f"{wildcard_provider_prefix}/{model_suffix if leading_is_listed_providers_own_prefix else model}"
 
 
 def expand_wildcard_deployments_for_model_info(

litellm/proxy/common_request_processing.py

@@ -724,7 +724,7 @@ def _router_defaults(llm_router: Router | None) -> Mapping[str, object]:
     return {name: value for name, value in _REQUEST_FIELDS.validate_python(configured).items() if value is not None}
 
 
-def _counted_drop_policies(
+def counted_drop_policies(
     data: Mapping[str, object], llm_router: Router | None, team_id: str | None
 ) -> tuple[DropPolicy, ...]:
     requested: Final = {**_router_defaults(llm_router), **data}
@@ -740,7 +740,7 @@ def _counted_drop_policies(
 def resolve_counted_tool_fields(
     data: Mapping[str, object], llm_router: Router | None, team_id: str | None
 ) -> Mapping[str, object] | ProxyException:
-    resolved: Final = resolve_tool_fields(data, lambda: _counted_drop_policies(data, llm_router, team_id))
+    resolved: Final = resolve_tool_fields(data, lambda: counted_drop_policies(data, llm_router, team_id))
     if isinstance(resolved, MalformedField):
         return ProxyException(
             message=resolved.message,
@@ -3206,7 +3206,7 @@ class ProxyBaseLLMRequestProcessing:
                         request_data=self.data,
                         request=request,
                     )
-                    if route_type == "aresponses":
+                    if route_type in ("aresponses", "aget_responses"):
                         selected_data_generator = (
                             ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership(
                                 original_stream_response=response,

litellm/proxy/common_utils/claude_code_marketplace_auth.py

@@ -0,0 +1,17 @@
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+_AUTH_REQUIRED_SETTING: Final[TypeAdapter[bool | None]] = TypeAdapter(bool | None)
+
+
+def marketplace_auth_required() -> bool:
+    from litellm.proxy.proxy_server import general_settings_view
+
+    try:
+        return (
+            _AUTH_REQUIRED_SETTING.validate_python(general_settings_view().get("claude_code_marketplace_auth_required"))
+            is True
+        )
+    except ValidationError:
+        return True

litellm/proxy/common_utils/codex_model_catalog.py

@@ -102,6 +102,7 @@ class CodexFallbackModel(LiteLLMBaseModel):
     experimental_supported_tools: tuple[()] = ()
     context_window: int | None
     base_instructions: str
+    supports_search_tool: bool = False
 
 
 class CodexStockUpgrade(LiteLLMBaseModel):
@@ -134,6 +135,7 @@ class CodexStockModel(LiteLLMBaseModel):
     upgrade: CodexStockUpgrade | None = None
     service_tiers: tuple[CodexServiceTier, ...] = ()
     default_service_tier: str | None = None
+    supports_search_tool: bool = False
     supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = ()
     context_window: int | None = None
 
@@ -160,6 +162,7 @@ class CodexCatalogRow:
     reasoning_effort: str | None = None
     display_name: str | None = None
     service_tiers: tuple[object, ...] = ()
+    search_tool_support: tuple[bool | None, ...] = ()
 
 
 @cache
@@ -252,6 +255,12 @@ def configured_service_tiers(
     return offered
 
 
+def offered_search_tool(per_deployment: Sequence[bool | None]) -> bool | None:
+    if False in per_deployment:
+        return False
+    return True if per_deployment and None not in per_deployment else None
+
+
 def _codex_can_drive(row: CodexCatalogRow) -> bool:
     return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES)
 
@@ -306,6 +315,7 @@ def _entry(
         {tier.id: tier for tier in (stock_model.service_tiers if stock_model is not None else ())}
     )
     configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers)
+    search_tool: Final = offered_search_tool(row.search_tool_support)
     if stock_model is None:
         return CodexFallbackModel(
             slug=row.id,
@@ -314,6 +324,7 @@ def _entry(
             service_tiers=configured if configured is not None else (),
             context_window=row.max_input_tokens,
             base_instructions=instructions,
+            supports_search_tool=search_tool is True,
         )
     upgrade: Final = (
         stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None
@@ -327,6 +338,7 @@ def _entry(
         "visibility": "list",
         "supported_in_api": True,
         "upgrade": upgrade,
+        **({} if search_tool is None else {"supports_search_tool": search_tool}),
         **(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}),
     }
     if configured is None:
@@ -433,6 +445,7 @@ def _catalog_row(
         upstream_model=llm_router.get_routable_upstream_model(lookup_id, team_id) if llm_router is not None else None,
         display_name=llm_router.get_configured_display_name(label_id) if llm_router is not None else None,
         service_tiers=llm_router.get_configured_service_tiers(lookup_id, team_id) if llm_router is not None else (),
+        search_tool_support=llm_router.get_search_tool_support(lookup_id, team_id) if llm_router is not None else (),
     )
 
 

litellm/proxy/common_utils/html_forms/native_client_consent.py

@@ -62,7 +62,7 @@ button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: p
 <p>Approving issues it a personal credential that expires within {CLI_JWT_EXPIRATION_HOURS} hours. <code>lite logout</code> stops it from being renewed. Only approve if you started this sign-in yourself.</p>
 <form method="post" action="{escape(complete_url)}">
 <input type="hidden" name="flow" value="{escape(flow_handle)}">
-{_team_field(teams)}
+{team_select_field(teams)}
 <div class="actions">
 <button type="submit" name="decision" value="deny" class="deny">Deny</button>
 <button type="submit" name="decision" value="approve" class="approve">Approve</button>
@@ -74,7 +74,7 @@ button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: p
 """
 
 
-def _team_field(teams: Sequence[tuple[str, str]]) -> str:
+def team_select_field(teams: Sequence[tuple[str, str]], *, require_choice: bool = False) -> str:
     if not teams:
         return ""
     if len(teams) == 1:
@@ -83,9 +83,12 @@ def _team_field(teams: Sequence[tuple[str, str]]) -> str:
             f'<input type="hidden" name="team_id" value="{escape(team_id)}">'
             f"<p>Requests are attributed to team <strong>{escape(team_label)}</strong>.</p>"
         )
+    placeholder: Final = '<option value="" disabled selected>Choose a team</option>' if require_choice else ""
     options: Final = "".join(
         f'<option value="{escape(team_id)}">{escape(team_label)}</option>' for team_id, team_label in teams
     )
+    required: Final = " required" if require_choice else ""
     return (
-        f'<label for="team_id">Attribute requests to team</label><select id="team_id" name="team_id">{options}</select>'
+        '<label for="team_id">Attribute requests to team</label>'
+        f'<select id="team_id" name="team_id"{required}>{placeholder}{options}</select>'
     )

litellm/proxy/db/check_migration.py

@@ -4,7 +4,7 @@ import os
 import subprocess
 from typing import Final
 
-from litellm._logging import print_startup_message, verbose_logger
+from litellm._logging import print_startup_message, print_startup_warning, verbose_logger
 
 
 def extract_sql_commands(diff_output: str) -> list[str]:
@@ -63,7 +63,7 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]:
             run_prisma,
         )
     except ImportError as e:
-        print_startup_message(
+        print_startup_warning(
             f"Skipping the migration diff: litellm-proxy-extras has no Prisma runner. Error: {e}",
         )
         return False, []
@@ -96,14 +96,14 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]:
         else:
             return False, []
     except subprocess.TimeoutExpired:
-        print_startup_message(
+        print_startup_warning(
             f"Timed out after {timeout}s generating the migration diff. "
             f"Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer.",
         )
         return False, []
     except subprocess.CalledProcessError as e:
         error_message: Final = f"Failed to generate migration diff. Error: {e.stderr}"
-        print_startup_message(error_message)
+        print_startup_warning(error_message)
         return False, []
 
 

litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py

@@ -421,7 +421,10 @@ class SpendLogCleanup:
         general_settings=None,
         redis_cache: RedisCache | None = None,
         partition_manager: SpendLogsPartitionManager | None = None,
+        *,
+        clock: Callable[[], float] = time.monotonic,
     ):
+        self._clock = clock
         self.retention_seconds: int | None = None
         self.partition_manager = partition_manager or SpendLogsPartitionManager()
         from litellm.proxy.proxy_server import general_settings as default_settings
@@ -586,7 +589,7 @@ class SpendLogCleanup:
 
     async def _probe_storage(self, prisma_client: PrismaClient, deadline: float) -> StorageProbe | None:
         """Read what the spend-log tables weigh from the catalog, or None when the probe fails or has no budget."""
-        if time.monotonic() >= deadline:
+        if self._clock() >= deadline:
             return None
         timeout_ms: Final = self._timeout_ms(deadline)
         try:
@@ -624,7 +627,7 @@ class SpendLogCleanup:
             stale.allocated_bytes,
             _ANALYZE_SQL,
         )
-        if time.monotonic() >= deadline:
+        if self._clock() >= deadline:
             return None
         timeout_ms: Final = self._timeout_ms(deadline)
         try:
@@ -653,7 +656,7 @@ class SpendLogCleanup:
 
     async def _load_size_state(self, prisma_client: PrismaClient, deadline: float) -> SizeCleanupState | None:
         """The reclaimable count saved by the last check, or None on the first check or when it cannot be read."""
-        if time.monotonic() >= deadline:
+        if self._clock() >= deadline:
             return None
         timeout_ms: Final = self._timeout_ms(deadline)
         try:
@@ -686,7 +689,7 @@ class SpendLogCleanup:
         the tool-index rows that expire with their spend logs. When fewer rows than that remain, the step is
         everything up to now.
         """
-        if time.monotonic() >= deadline:
+        if self._clock() >= deadline:
             return None
         timeout_ms: Final = self._timeout_ms(deadline)
         async with db_span("size_step_cutoff", "LiteLLM_SpendLogs"), bounded_tx(prisma_client, timeout_ms) as tx:
@@ -718,11 +721,11 @@ class SpendLogCleanup:
         Interpolating this into SQL is safe by construction: an int cannot carry
         SQL, and SET does not accept a bind parameter.
         """
-        remaining_ms: Final = int((deadline - time.monotonic()) * 1000)
+        remaining_ms: Final = int((deadline - self._clock()) * 1000)
         return max(1, min(int(self.batch_timeout_seconds * 1000), remaining_ms))
 
     @staticmethod
-    def _group_deadline(overall_deadline: float, groups_remaining: int) -> float:
+    def _group_deadline(overall_deadline: float, groups_remaining: int, now: float) -> float:
         """
         Give each pending cleanup group an equal share of the time left.
 
@@ -731,10 +734,9 @@ class SpendLogCleanup:
         """
         if groups_remaining == 1:
             return overall_deadline
-        current_time: Final = time.monotonic()
-        if current_time >= overall_deadline:
+        if now >= overall_deadline:
             return overall_deadline
-        return current_time + (overall_deadline - current_time) / groups_remaining
+        return now + (overall_deadline - now) / groups_remaining
 
     def _remaining_timeout_ms(self, deadline: float) -> RemainingTimeoutMs:
         """
@@ -748,7 +750,7 @@ class SpendLogCleanup:
         """
 
         def remaining() -> int | None:
-            return None if time.monotonic() >= deadline else self._timeout_ms(deadline)
+            return None if self._clock() >= deadline else self._timeout_ms(deadline)
 
         return remaining
 
@@ -841,7 +843,7 @@ class SpendLogCleanup:
         run_count = 0
         consecutive_failures = 0
         while True:
-            if time.monotonic() >= deadline:
+            if self._clock() >= deadline:
                 verbose_proxy_logger.info(
                     "Run budget exhausted during %s cleanup after %d rows; the next run resumes from here",
                     table_name,
@@ -858,13 +860,13 @@ class SpendLogCleanup:
                     prisma_client, cutoff_date, table_name, time_column, total_deleted, "batch_cap_reached", deadline
                 )
             # Find rows and delete them in one go without fetching to application
-            batch_started_at = time.monotonic()
+            batch_started_at = self._clock()
             try:
                 batch_result = await self._execute_delete_batch(
                     prisma_client, delete_sql, cutoff_date, table_name, deadline
                 )
             except Exception as batch_exc:
-                if time.monotonic() >= deadline:
+                if self._clock() >= deadline:
                     # The statement timeout was clamped to the budget that was
                     # left, so this batch was cancelled by the deadline itself.
                     # That is the bound working, not a database fault, and
@@ -920,7 +922,7 @@ class SpendLogCleanup:
 
             consecutive_failures = 0
             deleted_count = batch_result
-            SpendLogCleanupMetrics.record_batch(table_name, deleted_count, time.monotonic() - batch_started_at)
+            SpendLogCleanupMetrics.record_batch(table_name, deleted_count, self._clock() - batch_started_at)
             verbose_proxy_logger.info("Deleted %s %s rows in this batch", deleted_count, table_name)
 
             if deleted_count == 0:
@@ -957,7 +959,7 @@ class SpendLogCleanup:
         operator the backlog was not drained; the gauge simply keeps its value
         from the last run that finished inside its budget.
         """
-        if time.monotonic() >= deadline:
+        if self._clock() >= deadline:
             return TableCleanupResult(table_name=table_name, rows_deleted=rows_deleted, stop_reason=stop_reason)
         remaining: Final = await self._count_remaining(prisma_client, cutoff_date, table_name, time_column, deadline)
         if remaining is not None:
@@ -1070,7 +1072,7 @@ class SpendLogCleanup:
         # behind any long-running reader for as long as that reader lives, which
         # is the one way this job could still outlast its budget without bound.
         remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline)
-        if time.monotonic() >= deadline:
+        if self._clock() >= deadline:
             verbose_proxy_logger.info("Run budget already spent, skipping partition maintenance this run")
         elif self.general_settings.get(
             "use_spend_logs_partitioning", False
@@ -1082,7 +1084,7 @@ class SpendLogCleanup:
             verbose_proxy_logger.info("Dropped %d expired spend-log partitions: %s", len(dropped), dropped)
 
         logs_result: Final = await self._delete_old_logs(
-            prisma_client, cutoff_date, self._group_deadline(deadline, groups_remaining=2)
+            prisma_client, cutoff_date, self._group_deadline(deadline, groups_remaining=2, now=self._clock())
         )
         verbose_proxy_logger.info("Deleted %s logs", logs_result.rows_deleted)
 
@@ -1210,7 +1212,7 @@ class SpendLogCleanup:
         self, prisma_client: PrismaClient, deadline: float, still_over_target: Callable[[], Awaitable[bool]]
     ) -> tuple[str, ...]:
         remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline)
-        if time.monotonic() >= deadline or not self._setting("use_spend_logs_partitioning"):
+        if self._clock() >= deadline or not self._setting("use_spend_logs_partitioning"):
             return ()
         if not await self.partition_manager.is_partitioned(prisma_client, remaining_timeout_ms):
             return ()
@@ -1257,7 +1259,7 @@ class SpendLogCleanup:
         needed: Final = rows_still_needed(state.logs_deleted + state.index_deleted)
         if needed <= 0:
             return state.stopped("target_reached")
-        if time.monotonic() >= deadline:
+        if self._clock() >= deadline:
             return state.stopped("budget_exhausted")
         if state.batches_left <= 0:
             return state.stopped("batch_cap_reached")
@@ -1318,11 +1320,11 @@ class SpendLogCleanup:
             except Exception:  # noqa: BLE001  # retained observations are retried by the next cleanup job
                 verbose_proxy_logger.warning("Auto-router baseline retention remains pending")
         sessions_result: Final = await self._delete_old_autorouter_session_rows(
-            prisma_client, session_cutoff, self._group_deadline(deadline, 3)
+            prisma_client, session_cutoff, self._group_deadline(deadline, 3, self._clock())
         )
         verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_result.rows_deleted)
         user_sessions_result: Final = await self._delete_old_autorouter_user_session_rows(
-            prisma_client, session_cutoff, self._group_deadline(deadline, 2)
+            prisma_client, session_cutoff, self._group_deadline(deadline, 2, self._clock())
         )
         verbose_proxy_logger.info(
             "Deleted %s expired auto-router user session rollup rows", user_sessions_result.rows_deleted
@@ -1443,7 +1445,7 @@ class SpendLogCleanup:
 
     async def _run_passes(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> None:
         lock_acquired = False
-        run_started_at: Final = time.monotonic()
+        run_started_at: Final = self._clock()
         progress: Final = _RunProgress()
         progress_token: Final = _run_progress.set(progress)
         try:
@@ -1493,7 +1495,7 @@ class SpendLogCleanup:
                     SpendLogCleanupMetrics.record_run("skipped_locked")
                     return
 
-            deadline: Final = time.monotonic() + self.run_budget_seconds
+            deadline: Final = self._clock() + self.run_budget_seconds
             size_group: Final = int(active_size_rule is not None)
             configured_group_count: Final = (
                 int(delete_spend_logs and self.retention_seconds is not None)
@@ -1506,7 +1508,7 @@ class SpendLogCleanup:
             spend_log_results: Final = (
                 await self._clean_spend_log_tables(
                     prisma_client,
-                    self._group_deadline(deadline, configured_group_count),
+                    self._group_deadline(deadline, configured_group_count, self._clock()),
                 )
                 if delete_spend_logs and self.retention_seconds is not None
                 else ()
@@ -1526,7 +1528,7 @@ class SpendLogCleanup:
                 await self._clean_session_rollup(
                     prisma_client,
                     autorouter_retention_seconds,
-                    self._group_deadline(deadline, remaining_groups_after_spend_logs),
+                    self._group_deadline(deadline, remaining_groups_after_spend_logs, self._clock()),
                 )
                 if autorouter_retention_seconds is not None
                 else ()
@@ -1535,7 +1537,7 @@ class SpendLogCleanup:
                 await self._clean_health_checks(
                     prisma_client,
                     health_check_retention_seconds,
-                    self._group_deadline(deadline, remaining_groups_after_sessions),
+                    self._group_deadline(deadline, remaining_groups_after_sessions, self._clock()),
                 )
                 if health_check_retention_seconds is not None
                 else ()
@@ -1544,7 +1546,7 @@ class SpendLogCleanup:
                 await self._clean_daily_tag_spend(
                     prisma_client,
                     daily_tag_spend_retention_seconds,
-                    self._group_deadline(deadline, 1 + size_group),
+                    self._group_deadline(deadline, 1 + size_group, self._clock()),
                 )
                 if daily_tag_spend_retention_seconds is not None
                 else ()
@@ -1561,12 +1563,12 @@ class SpendLogCleanup:
             )
             outcome: Final = self._run_outcome(results)
             SpendLogCleanupMetrics.record_run(outcome)
-            self._log_run_summary(outcome, results, time.monotonic() - run_started_at, size_report)
+            self._log_run_summary(outcome, results, self._clock() - run_started_at, size_report)
 
         except asyncio.CancelledError:
             verbose_proxy_logger.error(
                 "Spend log cleanup cancelled after %.2fs (rows_deleted=%d, batches=%d); the next run resumes from here",
-                time.monotonic() - run_started_at,
+                self._clock() - run_started_at,
                 progress.rows_deleted,
                 progress.batches,
             )

litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py

@@ -4,6 +4,7 @@ from typing import Final
 
 from fastapi import APIRouter
 
+from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required
 from litellm.proxy.common_utils.html_forms.default_credentials_hint import should_hide_default_credentials_hint
 from litellm.proxy.mcp_server.stdio_gate import is_mcp_stdio_enabled
 from litellm.types.proxy.discovery_endpoints.ui_discovery_endpoints import (
@@ -51,4 +52,5 @@ async def get_ui_config():
         workers=proxy_config.worker_registry if is_control_plane else [],
         mcp_stdio_enabled=is_mcp_stdio_enabled(),
         use_si_terminology=use_si_terminology,
+        claude_code_marketplace_auth_required=marketplace_auth_required(),
     )

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -76,6 +76,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
     canonical_provider_batch_id,
 )
 from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
+from litellm.proxy.spend_tracking.input_tokens import tokenizer_models
 from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
 from litellm.router_utils.add_retry_fallback_headers import (
     ensure_response_additional_headers,
@@ -159,6 +160,12 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
     return resolve_model_group_alias(llm_router.model_group_alias, model)
 
 
+def _tokenizer_models_via_proxy_router(model: str) -> tuple[str, ...]:
+    from litellm.proxy.proxy_server import llm_router
+
+    return tokenizer_models(model, llm_router)
+
+
 def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None:
     from litellm.proxy.proxy_server import llm_router
 
@@ -884,6 +891,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         ] = _resolve_ptu_team_ceiling_via_proxy_router,
         fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings,
         force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings,
+        tokenizer_models_resolver: Callable[[str], tuple[str, ...]] = _tokenizer_models_via_proxy_router,
     ):
         self.internal_usage_cache = internal_usage_cache
         self._time_provider = time_provider or datetime.now
@@ -893,6 +901,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
         self._fail_closed_resolver = fail_closed_resolver
         self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver
+        self._tokenizer_models_resolver = tokenizer_models_resolver
         if self.internal_usage_cache.dual_cache.redis_cache is not None:
             self.batch_rate_limiter_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
                 BATCH_RATE_LIMITER_SCRIPT
@@ -4009,15 +4018,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         try:
             estimate: Final = max(
                 0,
-                int(
-                    token_counter(
-                        model=model or "",
-                        messages=countable_messages,
-                        text=selected_text,
-                        tools=countable_tools,
-                        tool_choice=countable_tool_choice,
-                        use_default_image_token_count=True,
+                *(
+                    int(
+                        token_counter(
+                            model=tokenizer_model,
+                            messages=countable_messages,
+                            text=selected_text,
+                            tools=countable_tools,
+                            tool_choice=countable_tool_choice,
+                            use_default_image_token_count=True,
+                        )
                     )
+                    for tokenizer_model in self._tokenizer_models_resolver(model or "")
                 ),
             )
             return estimate + audio_token_estimate

litellm/proxy/hooks/responses_id_security.py

@@ -31,7 +31,7 @@ if TYPE_CHECKING:
 
 
 _RESPONSES_API_PROVIDER_PREFIX: Final = "/openai"
-_RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
+_RESPONSES_API_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
 _ADDRESSED_RESPONSE_ID_FIELD_BY_CALL_TYPE: Final[Mapping[str, str]] = MappingProxyType(
     {
         "aresponses": "previous_response_id",
@@ -104,7 +104,7 @@ def _rewrite_advertised_id(
     return event
 
 
-def _is_responses_api_create_route(request_route: str | None) -> bool:
+def _streams_response_ids(request_route: str | None) -> bool:
     if request_route is None:
         return False
     canonical: Final = (
@@ -112,7 +112,8 @@ def _is_responses_api_create_route(request_route: str | None) -> bool:
         if request_route.startswith(_RESPONSES_API_PROVIDER_PREFIX + "/")
         else request_route
     )
-    return canonical in _RESPONSES_API_CREATE_ROUTES
+    parent, _, response_id = canonical.rpartition("/")
+    return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "")
 
 
 class ResponsesIDSecurity(CustomLogger):
@@ -339,7 +340,7 @@ class ResponsesIDSecurity(CustomLogger):
         async for chunk in response:
             if (
                 isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
-                and _is_responses_api_create_route(user_api_key_dict.request_route)
+                and _streams_response_ids(user_api_key_dict.request_route)
                 and not general_settings.get("disable_responses_id_security", False)
             ):
                 chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache)

litellm/proxy/management_endpoints/ui_sso.py

@@ -16,6 +16,7 @@ import json
 import os
 import re
 import secrets
+from collections import Counter
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from copy import deepcopy
 from html import escape
@@ -42,7 +43,7 @@ if TYPE_CHECKING:
 import jwt
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status
 from fastapi.responses import RedirectResponse
-from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator
+from pydantic import AliasPath, BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator
 
 import litellm
 from litellm._internal_context import with_service_target
@@ -106,6 +107,7 @@ from litellm.proxy.common_utils.html_forms.default_credentials_hint import shoul
 from litellm.proxy.common_utils.html_forms.jwt_display_template import (
     jwt_display_template,
 )
+from litellm.proxy.common_utils.html_forms.native_client_consent import team_select_field
 from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form
 from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
@@ -184,6 +186,8 @@ _CLI_SSO_START_RATE_LIMIT_WINDOW_SECONDS: Final = 60
 _CLI_SSO_START_RATE_LIMIT_MAX_ATTEMPTS: Final = 30
 _CLI_SSO_USER_CODE_ALPHABET: Final = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789"
 _CLI_SSO_LOGIN_ID_RE: Final = re.compile(r"^cli-[A-Za-z0-9_-]{12,124}$")
+CLI_SSO_BROWSER_TEAM_CHOICE: Final = "browser_team_choice"
+CLI_SSO_CHOSEN_TEAM_ID: Final = "chosen_team_id"
 _CLI_SSO_USER_CODE_RE = re.compile(rf"^[{_CLI_SSO_USER_CODE_ALPHABET}]{{4}}-[{_CLI_SSO_USER_CODE_ALPHABET}]{{4}}$")
 _CLI_SSO_SCALAR_TYPES: Final = (str, int, float, bool)
 _CLI_SSO_DEST_KEY_RE: Final = re.compile(r"^[A-Za-z0-9_.-]+$")
@@ -670,6 +674,7 @@ def _render_cli_sso_verification_page(
     verify_url: str,
     browser_complete_token: str,
     prefill_user_code: str | None = None,
+    team_choices: Sequence[tuple[str, str]] = (),
 ) -> str:
     escaped_verify_url: Final = escape(verify_url, quote=True)
     escaped_browser_complete_token: Final = escape(browser_complete_token, quote=True)
@@ -716,6 +721,15 @@ def _render_cli_sso_verification_page(
             letter-spacing: 0.08em;
             text-transform: uppercase;
           }}
+          select {{
+            box-sizing: border-box;
+            width: 100%;
+            padding: 12px;
+            margin-bottom: 18px;
+            border: 1px solid #cbd5e1;
+            border-radius: 6px;
+            font-size: 16px;
+          }}
           button {{
             margin-top: 16px;
             width: 100%;
@@ -735,6 +749,7 @@ def _render_cli_sso_verification_page(
           <p>{instructions}</p>
           <form method="post" action="{escaped_verify_url}">
             <input type="hidden" name="browser_complete_token" value="{escaped_browser_complete_token}" />
+            {team_select_field(team_choices, require_choice=True)}
             <label for="user_code">Verification code</label>
             <input id="user_code" name="user_code" autocomplete="one-time-code"{user_code_value_attr} required autofocus />
             <button type="submit">Continue</button>
@@ -823,13 +838,46 @@ async def cli_sso_complete(request: Request, login_id: str):
     ):
         raise HTTPException(status_code=400, detail="Invalid verification code")
 
-    flow["user_code_verified"] = True
-    _set_cli_sso_flow(login_id=login_id, cache=cli_sso_session_cache, flow=flow)
+    team_choice_flow: Final = _CliSsoTeamChoiceFlow.model_validate(flow)
+    chosen_team_id: Final = _browser_chosen_team_id(team_choice_flow, form_values)
+    if chosen_team_id is not None and chosen_team_id not in team_choice_flow.teams:
+        raise HTTPException(status_code=400, detail="Choose one of the teams you belong to")
+
+    _set_cli_sso_flow(
+        login_id=login_id,
+        cache=cli_sso_session_cache,
+        flow={
+            **flow,
+            "user_code_verified": True,
+            CLI_SSO_CHOSEN_TEAM_ID: chosen_team_id,
+            "session_data": (
+                flow["session_data"]
+                if chosen_team_id is None
+                else {
+                    **flow["session_data"],
+                    "teams": [chosen_team_id, *(team for team in team_choice_flow.teams if team != chosen_team_id)],
+                }
+            ),
+        },
+    )
 
     html_content: Final = render_cli_sso_success_page()
     return HTMLResponse(content=html_content, status_code=200)
 
 
+class _CliSsoTeamChoiceFlow(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    browser_team_choice: bool = Field(default=False, validation_alias=CLI_SSO_BROWSER_TEAM_CHOICE)
+    teams: tuple[str, ...] = Field(default=(), validation_alias=AliasPath("session_data", "teams"))
+
+
+def _browser_chosen_team_id(flow: _CliSsoTeamChoiceFlow, form_values: Mapping[str, Sequence[str]]) -> str | None:
+    if not flow.browser_team_choice:
+        return None
+    return (form_values.get("team_id") or [""])[0] or None
+
+
 def normalize_email(email: str | None) -> str | None:
     """
     Normalize email address to lowercase for consistent storage and comparison.
@@ -2416,6 +2464,18 @@ def _cli_sso_session_teams(team_details: Sequence[CliSsoTeamDetail]) -> list[str
     return [detail.team_id for detail in team_details if detail.team_id is not None]
 
 
+def _browser_team_choices(
+    flow: _CliSsoTeamChoiceFlow, team_details: Sequence[CliSsoTeamDetail]
+) -> tuple[tuple[str, str], ...]:
+    if not flow.browser_team_choice:
+        return ()
+    labeled: Final = tuple(
+        (detail.team_id, detail.team_alias or detail.team_id) for detail in team_details if detail.team_id is not None
+    )
+    label_counts: Final = Counter(label for _, label in labeled)
+    return tuple((team_id, label if label_counts[label] == 1 else f"{label} ({team_id})") for team_id, label in labeled)
+
+
 def selected_cli_sso_team_detail(team_details: object, team_id: str | None) -> CliSsoTeamDetail | None:
     """``None`` means the team's grants are unknown. An empty grant is a real value meaning unrestricted,
     so an unknown one must not be minted as empty."""
@@ -2484,6 +2544,7 @@ async def _complete_cli_sso_callback_session(
         request.scope.get("litellm_microsoft_interactive_subject"), resolved_user_id, prisma_client
     )
     resolved_teams: Final = _cli_sso_session_teams(team_details)
+    team_choices: Final = _browser_team_choices(_CliSsoTeamChoiceFlow.model_validate(flow), team_details)
     attribution_metadata: Final = build_cli_sso_attribution_metadata(result=result)
     if attribution_metadata:
         await _persist_cli_sso_user_metadata(
@@ -2521,6 +2582,7 @@ async def _complete_cli_sso_callback_session(
             verify_url=verify_url,
             browser_complete_token=browser_complete_token,
             prefill_user_code=prefill_user_code,
+            team_choices=team_choices,
         ),
         status_code=200,
     )

litellm/proxy/policy_engine/init_policies.py

@@ -11,7 +11,7 @@ from collections.abc import Mapping, Sequence
 from itertools import chain
 from typing import TYPE_CHECKING, Any, Final, Optional
 
-from litellm._logging import print_startup_message, verbose_proxy_logger
+from litellm._logging import print_startup_message, print_startup_warning, verbose_proxy_logger
 from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry
 from litellm.proxy.policy_engine.policy_registry import get_policy_registry
 from litellm.proxy.policy_engine.policy_validator import PolicyValidator
@@ -93,7 +93,7 @@ def _print_policies_on_startup(
         )
     else:
         print_startup_message("\n".join((header, *policy_lines)))
-        print_startup_message(
+        print_startup_warning(
             f"\n{_yellow_color_code}Warning: No policy_attachments configured. Policies will not be applied to any requests.{_reset_color_code}\n",
         )
     sys.stdout.flush()

litellm/proxy/proxy_cli.py

@@ -10,16 +10,17 @@ import sys
 import urllib.parse as urlparse
 from collections.abc import Callable, Iterable, Mapping, Sequence
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Annotated, Any, Final
 
 import click
 import httpx
 from click.core import ParameterSource
 from dotenv import load_dotenv
-from pydantic import ConfigDict
+from pydantic import BeforeValidator, ConfigDict, TypeAdapter
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
-from litellm._logging import print_startup_message
+from litellm._logging import StartupOutputSettings, print_startup_message, print_startup_warning, terminal_text
 from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
 from litellm.proxy.db.pgbouncer import (
     PgBouncerError,
@@ -205,7 +206,7 @@ def deprecated_v2_flag_passed_on_cli() -> bool:
 
 
 def _print_enforce_prisma_migration_check_notice() -> None:
-    print_startup_message(
+    print_startup_warning(
         "\033[1;33mLiteLLM Proxy: --enforce_prisma_migration_check is "
         "deprecated and has no effect, because the proxy always exits "
         "when database setup fails at startup. You can safely remove it.\033[0m",
@@ -227,6 +228,36 @@ def _print_or_hold_enforce_prisma_migration_check_notice(
     return print_once
 
 
+def _empty_as_none(value: object) -> object:
+    return value or None
+
+
+class _ConfigStartupOutputSettings(TypedDict, total=False):
+    litellm_settings: ReadOnly[Annotated[StartupOutputSettings | None, BeforeValidator(_empty_as_none)]]
+
+
+_CONFIG_STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(_ConfigStartupOutputSettings)
+
+
+def _apply_config_suppress_startup_output(litellm_settings: StartupOutputSettings | None) -> None:
+    if litellm_settings is None or litellm_settings.get("suppress_startup_output") is not True:
+        return
+    litellm.suppress_startup_output = True
+    os.environ["LITELLM_SUPPRESS_STARTUP_OUTPUT"] = "true"
+
+
+def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | None) -> None:
+    """Turn on the config's json_logs and suppress_startup_output before any worker starts. Workers that
+    uvicorn or granian spawn import litellm afresh and read both from the env at import, so they log JSON
+    and stay quiet from their first line instead of from when load_config reaches the key."""
+    _apply_config_suppress_startup_output(litellm_settings)
+    if litellm_settings is None or litellm_settings.get("json_logs") is not True:
+        return
+    litellm.json_logs = True
+    os.environ["JSON_LOGS"] = "true"
+    litellm.turn_on_json()
+
+
 class ProxyInitializationHelpers:
     @staticmethod
     def echo_litellm_version():
@@ -373,7 +404,7 @@ class ProxyInitializationHelpers:
             if "timeout_worker_healthcheck" in inspect.signature(uvicorn.Config.__init__).parameters:
                 uvicorn_args["timeout_worker_healthcheck"] = timeout_worker_healthcheck
             else:
-                print_startup_message(
+                print_startup_warning(
                     f"\033[1;33mLiteLLM Proxy: --timeout_worker_healthcheck "
                     f"requires uvicorn>=0.37.0, but installed uvicorn=={uvicorn.__version__}. "
                     f"Ignoring the flag.\033[0m",
@@ -396,7 +427,7 @@ class ProxyInitializationHelpers:
         import uvicorn
 
         if max_requests_before_restart is None:
-            print_startup_message(
+            print_startup_warning(
                 "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter "
                 "has no effect without --max_requests_before_restart\033[0m\n",
             )
@@ -404,7 +435,7 @@ class ProxyInitializationHelpers:
         if "limit_max_requests_jitter" in inspect.signature(uvicorn.Config.__init__).parameters:
             uvicorn_args["limit_max_requests_jitter"] = jitter
         else:
-            print_startup_message(
+            print_startup_warning(
                 f"\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter "
                 f"requires uvicorn>=0.41.0, but installed uvicorn=={uvicorn.__version__}. "
                 f"Ignoring the flag.\033[0m",
@@ -571,12 +602,12 @@ class ProxyInitializationHelpers:
 
         print_startup_message(f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Granian\033[0m\n")
         if max_requests_before_restart is not None:
-            print_startup_message(
+            print_startup_warning(
                 "\033[1;33mLiteLLM: --max_requests_before_restart is not supported by Granian "
                 "(Granian uses workers_lifetime in seconds, not a per-request limit).\033[0m\n",
             )
         if ciphers is not None:
-            print_startup_message("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n")
+            print_startup_warning("\033[1;33mLiteLLM: --ciphers is not applied when using --run_granian.\033[0m\n")
 
         kwargs: Final[dict[str, object]] = {
             "target": "litellm.proxy.proxy_server:app",
@@ -698,7 +729,7 @@ class ProxyInitializationHelpers:
             gunicorn_options["max_requests"] = max_requests_before_restart
         if max_requests_before_restart_jitter is not None:
             if max_requests_before_restart is None:
-                print_startup_message(
+                print_startup_warning(
                     "\033[1;33mLiteLLM Proxy: --max_requests_before_restart_jitter "
                     "has no effect without --max_requests_before_restart\033[0m\n",
                 )
@@ -739,7 +770,7 @@ class ProxyInitializationHelpers:
             with open(os.devnull, "w") as devnull:
                 subprocess.Popen(command, stdout=devnull, stderr=devnull)
         except Exception as e:
-            print_startup_message(f"""
+            print_startup_warning(f"""
                 LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve`
             """)
 
@@ -1325,6 +1356,19 @@ def run_server(
                 os.environ[k] = v
 
         litellm_settings = None
+        env_config_file: Final = os.environ.get("CONFIG_FILE_PATH")
+        if (
+            env_config_file
+            and os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is None
+            and ProxyConfig().is_yaml(config_file_path=env_config_file)
+        ):
+            import asyncio
+
+            _apply_config_suppress_startup_output(
+                _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python(
+                    asyncio.run(ProxyConfig().read_config_file(config_file_path=env_config_file))
+                ).get("litellm_settings")
+            )
         if config is not None:
             """
             Allow user to pass in db url via config
@@ -1342,17 +1386,9 @@ def run_server(
 
             ### LITELLM SETTINGS ###
             litellm_settings = _config.get("litellm_settings", None)
-            if (
-                litellm_settings is not None
-                and "json_logs" in litellm_settings
-                and litellm_settings["json_logs"] is True
-            ):
-                import litellm
-
-                litellm.json_logs = True
-                os.environ["JSON_LOGS"] = "true"
-
-                litellm.turn_on_json()
+            _apply_config_logging_settings(
+                _CONFIG_STARTUP_OUTPUT_SETTINGS.validate_python(_config).get("litellm_settings")
+            )
             if held_prisma_notice is not None:
                 held_prisma_notice()
             ### GENERAL SETTINGS ###
@@ -1450,7 +1486,9 @@ def run_server(
                 _bad_scheme = unsupported_db_scheme(_candidate_url)
                 if _bad_scheme is not None:
                     print(
-                        f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m",
+                        terminal_text(
+                            f"\033[1;31mLiteLLM Proxy: {unsupported_db_scheme_message(_db_env, _bad_scheme)}\033[0m"
+                        ),
                         file=sys.stderr,
                         flush=True,
                     )
@@ -1553,7 +1591,7 @@ def run_server(
                         env_value=os.getenv("USE_V2_MIGRATION_RESOLVER"),
                     )
                     if deprecated_v2_flag_passed_on_cli() and use_v2_resolver:
-                        print_startup_message(
+                        print_startup_warning(
                             "\033[1;33mLiteLLM Proxy: --use_v2_migration_resolver is "
                             "deprecated and has no effect, because the v2 migration "
                             "resolver is now the default. You can safely remove it. To "
@@ -1561,7 +1599,7 @@ def run_server(
                             "--use_legacy_migration_resolver.\033[0m",
                         )
                     if not use_v2_resolver:
-                        print_startup_message(
+                        print_startup_warning(
                             "\033[1;33mLiteLLM Proxy: Using the legacy (v1) migration "
                             "resolver. It performs the diff-and-force recovery that can "
                             "cause schema thrashing during rolling deploys where two "
@@ -1583,22 +1621,26 @@ def run_server(
                         # issues, and any `prisma db push` against a
                         # partitioned LiteLLM_SpendLogs.
                         print(
-                            f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m",
+                            terminal_text(f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m"),
                             file=sys.stderr,
                             flush=True,
                         )
                         sys.exit(2)
                     if not setup_ok:
                         print(
-                            "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. "
-                            "The proxy cannot start safely. Please check your database connection and migration status.\033[0m"
+                            terminal_text(
+                                "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. "
+                                "The proxy cannot start safely. Please check your database connection and migration status.\033[0m"
+                            )
                         )
                         sys.exit(1)
             else:
                 print(
-                    "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable "
-                    "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` "
-                    "or run a shipped LiteLLM image.\033[0m"
+                    terminal_text(
+                        "\033[1;31mLiteLLM Proxy: a database URL is set but the prisma CLI is neither on PATH nor importable "
+                        "as a package, so the database cannot be set up. Install it with `pip install 'litellm[extra_proxy]'` "
+                        "or run a shipped LiteLLM image.\033[0m"
+                    )
                 )
                 sys.exit(1)
         pgbouncer_settings: Final = PgBouncerSettings()
@@ -1609,8 +1651,10 @@ def run_server(
             )
             if isinstance(pooled_database_url, PgBouncerError):
                 print(
-                    f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer "
-                    f"could not start: {pooled_database_url.reason}\033[0m",
+                    terminal_text(
+                        f"\033[1;31mLiteLLM Proxy: LITELLM_PGBOUNCER_ENABLED is set but the in-container pgbouncer "
+                        f"could not start: {pooled_database_url.reason}\033[0m"
+                    ),
                     file=sys.stderr,
                     flush=True,
                 )
@@ -1675,7 +1719,7 @@ def run_server(
                 run_gunicorn=run_gunicorn is True,
             )
             if single_worker_warning is not None:
-                print_startup_message(f"\033[1;33m{single_worker_warning}\033[0m\n")
+                print_startup_warning(f"\033[1;33m{single_worker_warning}\033[0m\n")
         if run_gunicorn is False and run_hypercorn is False and run_granian is False:
             if limit_concurrency is not None:
                 uvicorn_args["limit_concurrency"] = limit_concurrency

litellm/proxy/proxy_server.py

@@ -100,7 +100,11 @@ from litellm.litellm_core_utils.litellm_logging import (
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
-from litellm.litellm_core_utils.token_counter import UnsupportedContentTypeError, offload_token_count
+from litellm.litellm_core_utils.token_counter import (
+    UnsupportedContentTypeError,
+    deployment_openai_model,
+    offload_token_count,
+)
 from litellm.llms.bedrock.count_tokens.kimi_k3 import (
     HUGGING_FACE_SOURCE,
     KimiK3Encodings,
@@ -306,6 +310,7 @@ import litellm._redis
 from litellm import Router
 from litellm._internal_context import service_target, with_service_target
 from litellm._logging import (
+    STARTUP_OUTPUT_SETTINGS,
     print_startup_message,
     redact_string,
     verbose_proxy_logger,
@@ -5801,6 +5806,12 @@ class ProxyConfig:
         # verbose_proxy_logger.debug(f"loaded config={json.dumps(config, indent=4)}")
         return config
 
+    async def read_config_file(self, config_file_path: str) -> Mapping[str, object]:
+        """The config file with its includes merged, without the bucket, database and secret steps of get_config."""
+        return await self._process_includes(
+            config=self._load_yaml_file(config_file_path), config_file_path=os.path.abspath(config_file_path)
+        )
+
     async def _process_includes(self, config: dict, config_file_path: str) -> dict:
         """
         Process includes by appending their contents to the main config
@@ -6425,13 +6436,8 @@ class ProxyConfig:
 
         search_tools_parsed: Final[list[SearchToolTypedDict]] = []
 
-        print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m")
-
         for search_tool in search_tools_raw:
-            # Display loaded search tool
             search_tool_name = search_tool.get("search_tool_name", "")
-            search_provider = search_tool.get("litellm_params", {}).get("search_provider", "")
-            print_startup_message(f"\033[32m    {search_tool_name} ({search_provider})\033[0m")
 
             # Handle os.environ/ variables in litellm_params
             litellm_params = search_tool.get("litellm_params", {})
@@ -6601,13 +6607,15 @@ class ProxyConfig:
             from litellm.types.integrations.prometheus import validate_caller_identity_settings
 
             validate_caller_identity_settings(litellm_settings)
+            if STARTUP_OUTPUT_SETTINGS.validate_python(litellm_settings).get("suppress_startup_output") is True:
+                litellm.suppress_startup_output = True
 
             # ANSI escape code for blue text
             blue_color_code: Final = "\033[94m"
             reset_color_code: Final = "\033[0m"
             for key, value in litellm_settings.items():
                 if key == "cache" and value is True:
-                    print_startup_message(f"{blue_color_code}\nSetting Cache on Proxy{reset_color_code}")
+                    print_startup_message(f"\n{blue_color_code}Setting Cache on Proxy{reset_color_code}")
                     from litellm.caching.caching import Cache
 
                     cache_params = {}
@@ -6922,6 +6930,8 @@ class ProxyConfig:
                     setattr(litellm, key, value)
                 elif key == "drop_params":
                     litellm.drop_params = drop_params_flag(value, "litellm_settings.drop_params", verbose_proxy_logger)
+                elif key == "suppress_startup_output":
+                    continue
                 else:
                     verbose_proxy_logger.debug(
                         "%s setting litellm.%s=%s%s",
@@ -6968,6 +6978,7 @@ class ProxyConfig:
                     for name in (
                         "vector_store_deny_by_default",
                         "search_tool_deny_by_default",
+                        "claude_code_marketplace_auth_required",
                         "vertex_ai_pass_through_drop_anthropic_betas",
                         "request_tag_mappings",
                     )
@@ -7277,6 +7288,13 @@ class ProxyConfig:
 
         ## SEARCH TOOLS SETTINGS
         search_tools: Final[list[SearchToolTypedDict] | None] = self.parse_search_tools(config)
+        if search_tools:
+            print_startup_message("\033[32mLiteLLM: Proxy initialized with Search Tools:\033[0m")
+            for tool in search_tools:
+                print_startup_message(
+                    f"\033[32m    {tool.get('search_tool_name', '')} "
+                    f"({tool.get('litellm_params', {}).get('search_provider', '')})\033[0m"
+                )
 
         ## SANDBOX TOOLS SETTINGS
         from litellm.sandbox.sandbox_tools import register_sandbox_tools
@@ -9679,12 +9697,9 @@ async def initialize(
         master_key, \
         user_custom_auth, \
         prisma_client
-    from litellm.proxy.common_utils.banner import LITELLM_BANNER, show_banner
+    from litellm.proxy.common_utils.banner import LITELLM_BANNER
 
-    if litellm.json_logs:
-        print_startup_message(LITELLM_BANNER)
-    else:
-        show_banner()
+    print_startup_message(f"\n{LITELLM_BANNER}\n")
     if os.getenv("LITELLM_DONT_SHOW_FEEDBACK_BOX", "").lower() != "true":
         generate_feedback_box()
     user_model = model
@@ -15542,7 +15557,7 @@ async def _count_request_tokens(
             count_source="local_estimate",
         )
     estimate: Final = await _local_token_estimate(
-        model=model_to_use,
+        model=deployment_openai_model(deployment_view) or model_to_use,
         prompt=prompt if counted_messages is None else None,
         messages=counted_messages,
         tools=counted_tools,

litellm/proxy/public_endpoints/provider_create_fields.json

@@ -4256,5 +4256,33 @@
       }
     ],
     "default_model_placeholder": "cursor/claude-4-sonnet"
+  },
+  {
+    "provider": "BIGMODEL",
+    "provider_display_name": "BigModel (Zhipu AI)",
+    "litellm_provider": "bigmodel",
+    "credential_fields": [
+      {
+        "key": "api_key",
+        "label": "BigModel (Zhipu AI) API Key",
+        "placeholder": null,
+        "tooltip": null,
+        "required": true,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "api_base",
+        "label": "API Base",
+        "placeholder": "https://open.bigmodel.cn",
+        "tooltip": null,
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": "https://open.bigmodel.cn"
+      }
+    ],
+    "default_model_placeholder": "bigmodel/glm-5.3"
   }
 ]

litellm/proxy/public_endpoints/public_endpoints.py

@@ -367,19 +367,23 @@ async def get_mcp_servers():
     tags=["public", "Claude Code Marketplace"],
 )
 async def public_skill_hub():
-    """Return enabled (public) Claude Code skills — no auth required."""
+    """Return enabled (public) Claude Code skills, or none when the marketplace requires a key."""
     from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import (
         MANIFEST_FIELDS,
         get_installation_preference,
-        get_prisma_client,
     )
+    from litellm.proxy.common_utils.claude_code_marketplace_auth import marketplace_auth_required
+    from litellm.proxy.proxy_server import prisma_client
     from litellm.types.proxy.claude_code_endpoints import (
         ListPluginsResponse,
         PluginListItem,
     )
 
+    if marketplace_auth_required():
+        return ListPluginsResponse(plugins=[], count=0)
+    if prisma_client is None:
+        raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
     try:
-        prisma_client: Final = await get_prisma_client()
         plugins: Final = await _plugin_table(prisma_client).find_many(where={"enabled": True})
         items: Final = []
         for plugin in plugins:
@@ -406,7 +410,8 @@ async def public_skill_hub():
             )
         return ListPluginsResponse(plugins=items, count=len(items))
     except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+        verbose_logger.exception("Error listing public skills: %s", e)
+        raise HTTPException(status_code=500, detail="Failed to list skills")
 
 
 @router.get(

litellm/proxy/response_api_endpoints/endpoints.py

@@ -39,6 +39,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata
 from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
 from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, first_validation_error
 from litellm.llms.openai.responses.guardrail_translation.handler import (
     OpenAIResponsesHandler,
@@ -85,6 +86,7 @@ from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
 from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_proxy
 from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction
 from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD
+from litellm.responses.retrieve_stream_query import RETRIEVE_INCLUDE_QUERY_KEYS, RETRIEVE_STREAM_QUERY_KEYS
 from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task
 from litellm.router import Router
 from litellm.types.llms.base import LiteLLMBaseModel
@@ -96,6 +98,7 @@ from litellm.types.llms.openai import (
 from litellm.types.responses.main import DeleteResponseResult
 from litellm.types.token_count import TokenCountSourceFields
 from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, TokenCountResponse
+from litellm.utils import get_base_model_from_metadata
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import ProxyLogging
@@ -104,11 +107,43 @@ router: Final = APIRouter()
 _RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None)
 _DECODED_JSON: Final = TypeAdapter(object)
 _PROXY_SERVER_REQUEST_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_RETRIEVE_QUERY_PARAMS: Final = tuple(sorted(RETRIEVE_INCLUDE_QUERY_KEYS | RETRIEVE_STREAM_QUERY_KEYS))
+_RETRIEVE_QUERY_OPENAPI: Final = {
+    "parameters": [
+        {
+            "name": "stream",
+            "in": "query",
+            "required": False,
+            "schema": {"type": "boolean"},
+            "description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)",
+        },
+        {
+            "name": "starting_after",
+            "in": "query",
+            "required": False,
+            "schema": {"type": "integer", "minimum": 0},
+            "description": "With stream=true, replay only the events after this sequence number",
+        },
+        {
+            "name": "include",
+            "in": "query",
+            "required": False,
+            "schema": {"type": "array", "items": {"type": "string"}},
+            "description": "Additional output data to include, forwarded to the provider",
+        },
+        {
+            "name": "include_obfuscation",
+            "in": "query",
+            "required": False,
+            "schema": {"type": "boolean"},
+            "description": "With stream=true, whether delta events carry obfuscation padding",
+        },
+    ]
+}
 
 _ResponseDocSchemas: TypeAlias = dict[int | str, dict[str, object]]  # fastapi's responses kwarg
 
-RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {200: {"model": ResponsesAPIResponse}}
-RESPONSES_API_CREATE_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {
+RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {
     200: {
         "model": ResponsesAPIResponse,
         "content": {
@@ -252,19 +287,19 @@ async def _resolve_cursor_model_variant_before_auth(request: Request) -> None:
     "/v1/responses",
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
-    responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS,
+    responses=RESPONSES_API_RESPONSE_SCHEMAS,
 )
 @router.post(
     "/responses",
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
-    responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS,
+    responses=RESPONSES_API_RESPONSE_SCHEMAS,
 )
 @router.post(
     "/openai/v1/responses",
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
-    responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS,
+    responses=RESPONSES_API_RESPONSE_SCHEMAS,
 )
 async def responses_api(
     request: Request,
@@ -537,7 +572,10 @@ async def cursor_model_list(
 
 def _dispatched_prompt_token_counter(logging_obj: LiteLLMLoggingObj) -> Callable[[], int]:
     return lambda: count_request_input_tokens(
-        model=str(logging_obj.model_call_details.get("model") or ""),
+        model=usage_tokenizer_model(
+            str(logging_obj.model_call_details.get("model") or ""),
+            get_base_model_from_metadata(logging_obj.model_call_details),
+        ),
         request_input=messages_since_newest_compaction(logging_obj.model_call_details.get("messages")),
         system_input=None,
     )
@@ -744,18 +782,21 @@ async def cursor_chat_completions(
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
     responses=RESPONSES_API_RESPONSE_SCHEMAS,
+    openapi_extra=_RETRIEVE_QUERY_OPENAPI,
 )
 @router.get(
     "/responses/{response_id}",
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
     responses=RESPONSES_API_RESPONSE_SCHEMAS,
+    openapi_extra=_RETRIEVE_QUERY_OPENAPI,
 )
 @router.get(
     "/openai/v1/responses/{response_id}",
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
     responses=RESPONSES_API_RESPONSE_SCHEMAS,
+    openapi_extra=_RETRIEVE_QUERY_OPENAPI,
 )
 async def get_response(
     response_id: str,
@@ -780,6 +821,10 @@ async def get_response(
     # Get provider response
     curl -X GET http://localhost:4000/v1/responses/resp_abc123 \
     -H "Authorization: Bearer $LITELLM_MASTER_KEY"
+
+    # Resume a background response's event stream after sequence number 7
+    curl -N "http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7" \
+    -H "Authorization: Bearer $LITELLM_MASTER_KEY"
     ```
     """
     from litellm.proxy.proxy_server import (
@@ -827,6 +872,14 @@ async def get_response(
     data: Final = await read_request_body(request=request)
     read_settles_billing: Final = not carries_client_side_overrides(data)
     data["response_id"] = response_id
+    retrieve_query: Final = {
+        key: values[0] if len(values) == 1 else values
+        for key in _RETRIEVE_QUERY_PARAMS
+        if (values := request.query_params.getlist(key))
+    }
+    if retrieve_query:
+        body_extra_query: Final[object] = data.get("extra_query")
+        data["extra_query"] = {**(body_extra_query if isinstance(body_extra_query, dict) else {}), **retrieve_query}
     processor: Final = ProxyBaseLLMRequestProcessing(data=data)
     try:
         response: Final[object] = await processor.base_process_llm_request(
@@ -838,7 +891,12 @@ async def get_response(
             llm_router=llm_router,
             general_settings=general_settings,
             proxy_config=proxy_config,
-            select_data_generator=select_data_generator,
+            select_data_generator=partial(
+                _streamed_retrieval_data_generator,
+                partial(select_data_generator, responses_stream_errors=True),
+                response_id,
+                read_settles_billing,
+            ),
             model=None,
             user_model=user_model,
             user_temperature=user_temperature,
@@ -912,12 +970,24 @@ async def _bill_finished_background_response(
     fastapi_response: Response,
     read_settles_billing: bool,
 ) -> None:
+    _report_charge(
+        await _settle_finished_background_read(response, response_id, user_api_key_dict, read_settles_billing),
+        fastapi_response,
+    )
+
+
+async def _settle_finished_background_read(
+    response: object,
+    response_id: str,
+    user_api_key_dict: UserAPIKeyAuth,
+    read_settles_billing: bool,
+) -> "_BilledCharge | None":
     from litellm.proxy.proxy_server import proxy_logging_obj
 
     if not isinstance(response, ResponsesAPIResponse) or not is_background_response(response):
-        return
+        return None
     if response.status not in TERMINAL_RESPONSE_STATUSES:
-        return
+        return None
     managed_files_hook: Final = _managed_files_hook(proxy_logging_obj)
     track_untracked: Final = (
         partial(
@@ -928,12 +998,11 @@ async def _bill_finished_background_response(
         if managed_files_hook is not None
         else None
     )
-    charge: Final = await _settle_background_billing(
+    return await _settle_background_billing(
         (lambda cost_check: cost_check.bill_if_finished(response_id, response, track_untracked))
         if read_settles_billing
         else (lambda cost_check: cost_check.bill_from_deployment(response_id, track_untracked))
     )
-    _report_charge(charge, fastapi_response)
 
 
 class _BilledCharge(Protocol):
@@ -1083,27 +1152,53 @@ def _streamed_background_data_generator(
         user_api_key_dict=user_api_key_dict,
         request_tags=_background_response_spend_tags(request_data),
     )
-    return _bill_streamed_background_response(response, chunks, track_untracked)
+    return _bill_streamed_background_response(
+        response, chunks, partial(_bill_finished_stream, track_untracked=track_untracked)
+    )
+
+
+def _streamed_retrieval_data_generator(
+    select_data_generator: Callable[..., AsyncIterator[str | bytes]],
+    response_id: str,
+    read_settles_billing: bool,
+    response: object,
+    user_api_key_dict: UserAPIKeyAuth,
+    request_data: Mapping[str, object],
+    request: Request | None = None,
+) -> AsyncIterator[str | bytes]:
+    chunks: Final = select_data_generator(
+        response=response, user_api_key_dict=user_api_key_dict, request_data=request_data, request=request
+    )
+    return _bill_streamed_background_response(
+        response,
+        chunks,
+        partial(
+            _settle_finished_background_read,
+            response_id=response_id,
+            user_api_key_dict=user_api_key_dict,
+            read_settles_billing=read_settles_billing,
+        ),
+    )
 
 
 async def _bill_streamed_background_response(
     stream: object,
     chunks: AsyncIterator[str | bytes],
-    track_untracked: Callable[[str, ResponsesAPIResponse], Awaitable[bool]],
+    bill: Callable[[ResponsesAPIResponse], Awaitable[object]],
 ) -> AsyncIterator[str | bytes]:
     async for chunk in chunks:
         completed: object = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream)
         if not isinstance(completed, ResponsesAPIResponse):
             yield chunk
             continue
-        await _bill_finished_stream(completed, track_untracked)
+        await bill(completed)
         yield chunk
         async for remaining_chunk in chunks:
             yield remaining_chunk
         return
     late_completed: Final = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream)
     if isinstance(late_completed, ResponsesAPIResponse):
-        await _bill_finished_stream(late_completed, track_untracked)
+        await bill(late_completed)
 
 
 async def _bill_finished_stream(

litellm/proxy/spend_tracking/budget_reservation.py

@@ -7,6 +7,7 @@ import time
 from collections.abc import AsyncIterator, Mapping, Sequence
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
+from itertools import chain
 from types import MappingProxyType
 from typing import Final, NoReturn, SupportsFloat, SupportsIndex, SupportsInt, cast
 
@@ -39,7 +40,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
     tag_cache_key,
     team_membership_reservation_cache_key,
 )
-from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model
+from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model, tokenizer_models
 from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 from litellm.router import Router
@@ -1599,7 +1600,19 @@ async def count_request_input_tokens(
     models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router)
     if not models:
         return MappingProxyType({})
-    return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models)
+    counted_as: Final = MappingProxyType({model: tokenizer_models(model, llm_router) for model in models})
+    counts: Final = await count_input_tokens(
+        request_body=request_body,
+        raw_body=raw_body,
+        models=tuple(dict.fromkeys(chain.from_iterable(counted_as.values()))),
+    )
+    return MappingProxyType(
+        {
+            model: count
+            for model, names in counted_as.items()
+            if (count := max((counts[name] for name in names if name in counts), default=None)) is not None
+        }
+    )
 
 
 def _estimate_input_tokens(

litellm/proxy/spend_tracking/input_tokens.py

@@ -15,14 +15,16 @@ import asyncio
 import json
 from collections.abc import Mapping, Sequence
 from types import MappingProxyType
-from typing import Final
+from typing import TYPE_CHECKING, Final
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.token_counter import served_openai_models, unresolved_tokenizer_model
 from litellm.responses.litellm_completion_transformation.compaction_items import (
     billable_input_since_newest_compaction,
     holds_compaction_summary,
 )
+from litellm.router_utils.common_utils import resolve_model_group_alias
 from litellm.rust_bridge import runtime
 from litellm.rust_bridge.catalog import Route, RouteContext
 from litellm.rust_bridge.token_counter import (
@@ -33,6 +35,9 @@ from litellm.rust_bridge.token_counter import (
     rust_tokenizer,
 )
 
+if TYPE_CHECKING:
+    from litellm.router import Router
+
 TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000
 
 _INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice")
@@ -47,6 +52,25 @@ def _approximate_input_size(request_body: Mapping[str, object]) -> int:
     return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS)
 
 
+def tokenizer_models(model: str, llm_router: Router | None) -> tuple[str, ...]:
+    """The names to count a request for `model` with, keeping the largest count.
+
+    An alias resolves to its target group. A group served only by OpenAI models counts as those models, so it gets
+    their tokenizer. Any other group counts as its OpenAI models, if any, and as the requested name with the encoding
+    tiktoken gives that name as written, so its other deployments keep that count without loading their vendor's
+    tokenizer. A name the router does not index counts as written."""
+    if llm_router is None:
+        return (model,)
+    group: Final = resolve_model_group_alias(llm_router.model_group_alias, model) or model
+    served: Final = served_openai_models(llm_router.deployments_named(group) or ())
+    if not served:
+        return (model,)
+    openai_served: Final = tuple(served_model for served_model in served if served_model is not None)
+    if len(openai_served) == len(served):
+        return openai_served
+    return (unresolved_tokenizer_model(model), *openai_served)
+
+
 async def count_input_tokens(
     request_body: dict,
     raw_body: bytes | None,

litellm/proxy/utils.py

@@ -168,7 +168,7 @@ from litellm.litellm_core_utils.served_output_texts import (
     record_served_output_texts,
     served_stream_output_texts,
 )
-from litellm.litellm_core_utils.token_counter import offload_token_count
+from litellm.litellm_core_utils.token_counter import offload_token_count, usage_tokenizer_model
 from litellm.llms import load_guardrail_translation_mappings
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.proxy._types import (
@@ -1139,7 +1139,9 @@ def _estimated_prompt_usage(
         None,
     )
     return _estimate_dispatched_failure_usage(
-        model=str(model_call_details.get("model") or ""),
+        model=usage_tokenizer_model(
+            str(model_call_details.get("model") or ""), get_base_model_from_metadata(model_call_details)
+        ),
         request_input=(
             messages_since_newest_compaction(dispatched_input)
             if dispatched_call_type in _RESPONSES_CALL_TYPES

litellm/realtime_api/main.py

@@ -31,7 +31,12 @@ from litellm.types.realtime import (
 )
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import CallTypes, LlmProviders
-from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference
+from litellm.utils import (
+    ModelInfoLookupMiss,
+    ProviderConfigManager,
+    lookup_model_info_for_registration,
+    resolve_api_key_secret_reference,
+)
 
 from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
 from ..litellm_core_utils.credential_accessor import CredentialAccessor
@@ -42,8 +47,13 @@ from ..llms.azure.common_utils import get_azure_ad_token
 from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client
 from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed
 from ..llms.custom_httpx.http_handler import realtime_ssl_for_url
+from ..llms.openai.common_utils import is_openai_backed_api_base
 from ..llms.openai.live.handler import OpenAILiveSessions
-from ..llms.openai.realtime.handler import OpenAIRealtime
+from ..llms.openai.realtime.handler import (
+    OpenAIRealtime,
+    confirm_gateway_session_started,
+    confirm_session_started,
+)
 from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async
 from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model
 from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config
@@ -740,6 +750,18 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str)
     return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ())
 
 
+_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions", "/v1/live")
+_GATEWAY_FIRST_EVENT_WAIT_SECONDS: Final = 3.0
+
+
+def _cost_map_lists_outside_realtime(model: str) -> bool:
+    model_info: Final = lookup_model_info_for_registration(model, "openai")
+    endpoints: Final = (
+        () if isinstance(model_info, ModelInfoLookupMiss) else model_info.get("supported_endpoints") or ()
+    )
+    return bool(endpoints) and not any(endpoint.startswith(_REALTIME_SERVED_ENDPOINTS) for endpoint in endpoints)
+
+
 _TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"}
 
 
@@ -785,6 +807,7 @@ async def realtime_health_check(
     api_version: str | None = None,
     realtime_protocol: str | None = None,
     model_params: Mapping[str, object] | None = None,
+    gateway_first_event_wait_seconds: float = _GATEWAY_FIRST_EVENT_WAIT_SECONDS,
 ) -> bool:
     """
     Health check for realtime API - tries connection to the realtime API websocket
@@ -800,9 +823,16 @@ async def realtime_health_check(
             without the OpenAI-Beta header is bridged to, with transcription-only models adding intent=transcription
 
     Returns:
-        bool - True if connection is successful, False otherwise
+        bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an
+            error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
+            the failure in its first server event. Behind any other OpenAI api_base the first event is read for
+            `gateway_first_event_wait_seconds` only, and a gateway that stays silent that long counts as healthy.
+            An `invalid_model` error also counts as healthy unless litellm's
+            cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live
+            and transcription models are served through other endpoints. The listed `mode` is not read, since a
+            deployment's own `mode: realtime` replaces it when the router registers the deployment
     Raises:
-        Exception - if the connection is not successful
+        Exception - if the connection is not successful or if OpenAI's first server event is any other error
     """
     import websockets
 
@@ -822,10 +852,11 @@ async def realtime_health_check(
         str | None, api_version or resolved_params.get("api_version")
     )
     url: str | None = None
+    openai_api_base: Final = resolved_api_base or litellm.api_base or "https://api.openai.com/"
     health_check_api_key: Final = (
         await resolve_openai_bearer_token_async(
             static_api_key=openai_static_api_key(resolved_api_key),
-            api_base=resolved_api_base,
+            api_base=openai_api_base,
             litellm_params=resolved_params,
         )
         if custom_llm_provider == LlmProviders.OPENAI.value
@@ -850,10 +881,7 @@ async def realtime_health_check(
             query_params=azure_query_params,
         )
     elif custom_llm_provider == "openai":
-        url = openai_realtime.construct_url(
-            api_base=resolved_api_base or "https://api.openai.com/",
-            query_params={"model": model},
-        )
+        url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model})
     elif custom_llm_provider == "xai":
         url = xai_realtime.construct_url(
             api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model}
@@ -899,8 +927,23 @@ async def realtime_health_check(
         additional_headers=auth_headers,
         max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
         ssl=realtime_ssl_for_url(url),
-    ):
-        return True
+    ) as connection:
+        if custom_llm_provider != "openai":
+            return True
+        if not is_openai_backed_api_base(url):
+            return await confirm_gateway_session_started(
+                connection,
+                model,
+                sent_credentials=bool(auth_headers),
+                model_listed_outside_realtime=_cost_map_lists_outside_realtime(model),
+                first_event_wait_seconds=gateway_first_event_wait_seconds,
+            )
+        return await confirm_session_started(
+            connection,
+            model,
+            sent_credentials=bool(auth_headers),
+            model_listed_outside_realtime=_cost_map_lists_outside_realtime(model),
+        )
 
 
 _realtime_health_check = realtime_health_check

litellm/responses/additional_tools.py

@@ -1,5 +1,8 @@
-from collections.abc import Sequence
+import json
+from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass
+from itertools import groupby
+from types import MappingProxyType
 from typing import Final, cast  # noqa: TID251  # validating the openai tool union strips vendor keys from raw tools
 
 from pydantic import ValidationError
@@ -15,8 +18,8 @@ class _InputItemType(LiteLLMBaseModel):
     type: str = ""
 
 
-class _AdditionalToolsItem(LiteLLMBaseModel):
-    tools: tuple[dict[str, object], ...] = ()
+class _ToolList(LiteLLMBaseModel):
+    tools: tuple[Mapping[str, object], ...] = ()
 
 
 @dataclass(frozen=True, slots=True)
@@ -33,12 +36,54 @@ def _is_additional_tools_item(item: object) -> bool:
         return False
 
 
-def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]:
+def _tool_list_of(value: object) -> tuple[Mapping[str, object], ...]:
     try:
-        parsed: Final = _AdditionalToolsItem.model_validate(item)
+        return _ToolList.model_validate(value).tools
     except ValidationError:
         return ()
-    return tuple(cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) for tool in parsed.tools)
+
+
+def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]:
+    return tuple(cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) for tool in _tool_list_of(item))
+
+
+def _tool_identity(tool: Mapping[str, object]) -> str:
+    name: Final = tool.get("name")
+    if isinstance(name, str) and name:
+        return f"{tool.get('type')}:{name}"
+    return json.dumps(tool, sort_keys=True, default=str)
+
+
+def _members_of_copies(copies: Sequence[Mapping[str, object]]) -> Iterator[Mapping[str, object]]:
+    for namespace in copies:
+        yield from _tool_list_of(namespace)
+
+
+def _merged(copies: Sequence[Mapping[str, object]]) -> Mapping[str, object]:
+    newest: Final = copies[-1]
+    if newest.get("type") != "namespace" or len(copies) == 1:
+        return newest
+    members: Final = tuple(_members_of_copies(copies))
+    return {**newest, "tools": list(_unique_tools(members))} if members else newest
+
+
+def _identity_of(keyed_tool: tuple[str, Mapping[str, object]]) -> str:
+    return keyed_tool[0]
+
+
+def _unique_tools(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
+    """Codex re-sends its tool list in the input on every turn, so the same tool arrives many times.
+    Keep each tool once, at its first position, with the newest copy's definition, and merge the
+    members of a namespace sent more than once."""
+    keyed: Final = tuple((_tool_identity(tool), tool) for tool in tools)
+    copies_by_identity: Final = MappingProxyType(
+        {
+            identity: tuple(tool for _, tool in group)
+            for identity, group in groupby(sorted(keyed, key=_identity_of), key=_identity_of)
+        }
+    )
+    first_seen_order: Final = tuple(dict.fromkeys(identity for identity, _ in keyed))
+    return tuple(_merged(copies_by_identity[identity]) for identity in first_seen_order)
 
 
 def hoist_additional_tools(
@@ -58,4 +103,8 @@ def hoist_additional_tools(
         len(items),
     )
     remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)]
-    return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted)
+    unique: Final = tuple(
+        cast("ALL_RESPONSES_API_TOOL_PARAMS", tool)  # cast-ok: a merged namespace is rebuilt from the same tool dicts
+        for tool in _unique_tools((*existing, *hoisted))
+    )
+    return HoistedAdditionalTools(input=remaining_input, tools=unique, hoisted=hoisted)

litellm/responses/litellm_completion_transformation/client_tool_search.py

@@ -0,0 +1,268 @@
+import json
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from itertools import chain
+from types import MappingProxyType
+from typing import Final, cast  # noqa: TID251  # loaded tool specs are forwarded as the client sent them
+
+from openai.types.responses.response_input_param import (
+    FunctionCallOutput,
+    ResponseFunctionToolCallParam,
+    ResponseInputItemParam,
+)
+from pydantic import ConfigDict, TypeAdapter, ValidationError
+
+from litellm.types.llms.base import LiteLLMBaseModel
+from litellm.types.llms.openai import (
+    ALL_RESPONSES_API_TOOL_PARAMS,
+    ChatCompletionToolParam,
+    ChatCompletionToolParamFunctionChunk,
+    ResponseInputParam,
+)
+
+from .custom_tools import function_chat_tool
+from .tool_search import CLIENT_TOOL_SEARCH_FUNCTION_NAME, has_client_tool_search
+
+__all__ = [
+    "ClientToolSearchBridge",
+    "bridge_client_tool_search",
+    "client_tool_search_chat_tool",
+    "client_tool_search_loaded_tools",
+]
+
+_DEFAULT_SEARCH_DESCRIPTION: Final = (
+    "Search the deferred tools by keyword. The matching tools are loaded and become callable on the next turn."
+)
+_DEFAULT_SEARCH_PARAMETERS: Final = MappingProxyType(
+    {
+        "type": "object",
+        "properties": {
+            "query": {"type": "string", "description": "Search query for deferred tools."},
+            "limit": {"type": "integer", "description": "Maximum number of tools to return. Defaults to 8."},
+        },
+        "required": ["query"],
+        "additionalProperties": False,
+    }
+)
+
+
+class _ClientToolSearchItem(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: str = ""
+    execution: str | None = None
+    call_id: str | None = None
+    arguments: object = None
+    tools: tuple[Mapping[str, object], ...] = ()
+
+
+class _ToolSpec(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: str = ""
+    name: str = ""
+    description: str = ""
+    defer_loading: object = None
+    tools: tuple[Mapping[str, object], ...] = ()
+
+
+class _ClientToolSearchTool(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    description: str = ""
+    parameters: Mapping[str, object] | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class _LoadedTool:
+    namespace: str | None
+    namespace_description: str
+    spec: Mapping[str, object]
+
+    @property
+    def callable_name(self) -> str:
+        name: Final = str(self.spec.get("name") or "")
+        return name if self.namespace is None else f"{self.namespace}__{name}"
+
+
+@dataclass(frozen=True, slots=True)
+class ClientToolSearchBridge:
+    input: str | ResponseInputParam
+    tools: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]
+    bridged: bool
+
+
+def client_tool_search_chat_tool(tool: Mapping[str, object]) -> ChatCompletionToolParam:
+    spec: Final = _ClientToolSearchTool.model_validate(tool)
+    function: Final = ChatCompletionToolParamFunctionChunk(
+        name=CLIENT_TOOL_SEARCH_FUNCTION_NAME,
+        description=spec.description or _DEFAULT_SEARCH_DESCRIPTION,
+        parameters=dict(spec.parameters or _DEFAULT_SEARCH_PARAMETERS),
+    )
+    return function_chat_tool(function, None, False)
+
+
+def _client_tool_search_item(item: object) -> _ClientToolSearchItem | None:
+    try:
+        parsed: Final = _ClientToolSearchItem.model_validate(item)
+    except ValidationError:
+        return None
+    if parsed.type not in ("tool_search_call", "tool_search_output") or parsed.execution != "client":
+        return None
+    return parsed
+
+
+_INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...])
+
+
+def _searched_tools(item: object) -> tuple[Mapping[str, object], ...]:
+    parsed: Final = _client_tool_search_item(item)
+    if parsed is None or parsed.type != "tool_search_output" or not parsed.call_id:
+        return ()
+    return parsed.tools
+
+
+def client_tool_search_loaded_tools(input: object) -> tuple[Mapping[str, object], ...]:
+    if not isinstance(input, (list, tuple)):
+        return ()
+    return tuple(chain.from_iterable(map(_searched_tools, _INPUT_ITEMS.validate_python(input))))
+
+
+def _tool_spec(tool: object) -> _ToolSpec | None:
+    try:
+        return _ToolSpec.model_validate(tool)
+    except ValidationError:
+        return None
+
+
+def _as_tool_param(spec: Mapping[str, object]) -> ALL_RESPONSES_API_TOOL_PARAMS:
+    return cast("ALL_RESPONSES_API_TOOL_PARAMS", spec)  # cast-ok: client tool specs are forwarded as sent
+
+
+def _without_deferral(spec: Mapping[str, object]) -> Mapping[str, object]:
+    return {key: value for key, value in spec.items() if key != "defer_loading"}
+
+
+def _loaded_tools(tool: Mapping[str, object]) -> tuple[_LoadedTool, ...]:
+    spec: Final = _tool_spec(tool)
+    if spec is None:
+        return ()
+    if spec.type != "namespace":
+        return (_LoadedTool(namespace=None, namespace_description="", spec=_without_deferral(tool)),)
+    return tuple(
+        _LoadedTool(namespace=spec.name, namespace_description=spec.description, spec=_without_deferral(member))
+        for member in spec.tools
+    )
+
+
+def _all_loaded_tools(tools: Iterable[Mapping[str, object]]) -> tuple[_LoadedTool, ...]:
+    return tuple(chain.from_iterable(map(_loaded_tools, tools)))
+
+
+def _callable_names(tool: object) -> tuple[str, ...]:
+    spec: Final = _tool_spec(tool)
+    if spec is None:
+        return ()
+    if spec.type != "namespace":
+        return (spec.name,)
+    return tuple(f"{spec.name}__{member.get('name') or ''}" for member in spec.tools)
+
+
+def _first_occurrences(loaded: Sequence[_LoadedTool], taken: frozenset[str]) -> tuple[_LoadedTool, ...]:
+    first_index: Final = {tool.callable_name: index for index, tool in reversed(tuple(enumerate(loaded)))}
+    return tuple(
+        tool
+        for index, tool in enumerate(loaded)
+        if first_index[tool.callable_name] == index and tool.callable_name not in taken
+    )
+
+
+def _regrouped(loaded: Sequence[_LoadedTool]) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]:
+    namespaces: Final = {tool.namespace: tool.namespace_description for tool in reversed(loaded) if tool.namespace}
+    ordered_namespaces: Final = tuple(dict.fromkeys(tool.namespace for tool in loaded if tool.namespace))
+    top_level: Final = tuple(tool.spec for tool in loaded if tool.namespace is None)
+    grouped: Final = tuple(
+        {
+            "type": "namespace",
+            "name": namespace,
+            "description": namespaces[namespace],
+            "tools": tuple(tool.spec for tool in loaded if tool.namespace == namespace),
+        }
+        for namespace in ordered_namespaces
+    )
+    return tuple(_as_tool_param(tool) for tool in (*top_level, *grouped))
+
+
+def _is_deferred(tool: object) -> bool:
+    spec: Final = _tool_spec(tool)
+    return spec is not None and spec.defer_loading is True
+
+
+def _without_deferred(tool: ALL_RESPONSES_API_TOOL_PARAMS) -> ALL_RESPONSES_API_TOOL_PARAMS | None:
+    spec: Final = _tool_spec(tool)
+    if spec is None:
+        return tool
+    if spec.type != "namespace":
+        return None if spec.defer_loading is True else tool
+    eager_members: Final = tuple(member for member in spec.tools if not _is_deferred(member))
+    if len(eager_members) == len(spec.tools):
+        return tool
+    if not eager_members:
+        return None
+    return _as_tool_param({**tool, "tools": eager_members})
+
+
+def _search_output_text(loaded: Sequence[_LoadedTool]) -> str:
+    if not loaded:
+        return "No tools matched the search."
+    return "Loaded these tools, which are now callable: " + ", ".join(tool.callable_name for tool in loaded)
+
+
+def _search_arguments_json(arguments: object) -> str:
+    return arguments if isinstance(arguments, str) else json.dumps(arguments or {})
+
+
+def _bridged_item(item: ResponseInputItemParam) -> ResponseInputItemParam | None:
+    parsed: Final = _client_tool_search_item(item)
+    if parsed is None:
+        return item
+    if not parsed.call_id:
+        return None
+    if parsed.type == "tool_search_call":
+        return ResponseFunctionToolCallParam(
+            type="function_call",
+            call_id=parsed.call_id,
+            name=CLIENT_TOOL_SEARCH_FUNCTION_NAME,
+            arguments=_search_arguments_json(parsed.arguments),
+        )
+    return FunctionCallOutput(
+        type="function_call_output", call_id=parsed.call_id, output=_search_output_text(_all_loaded_tools(parsed.tools))
+    )
+
+
+def bridge_client_tool_search(
+    input: str | ResponseInputParam,
+    tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None,
+    loaded_earlier: Sequence[Mapping[str, object]] = (),
+) -> ClientToolSearchBridge:
+    existing: Final = tuple(tools or ())
+    searches_on_client: Final = has_client_tool_search(existing)
+    items: Final = () if isinstance(input, str) else tuple(input)
+    search_items: Final = tuple(parsed for item in items if (parsed := _client_tool_search_item(item)) is not None)
+    if not searches_on_client and not search_items:
+        return ClientToolSearchBridge(input=input, tools=existing, bridged=False)
+    eager: Final = (
+        tuple(kept for tool in existing if (kept := _without_deferred(tool)) is not None)
+        if searches_on_client
+        else existing
+    )
+    loaded: Final = _all_loaded_tools((*loaded_earlier, *client_tool_search_loaded_tools(input)))
+    taken: Final = frozenset(chain.from_iterable(map(_callable_names, eager)))
+    bridged_input: Final = (
+        input if isinstance(input, str) else [bridged for item in input if (bridged := _bridged_item(item)) is not None]
+    )
+    return ClientToolSearchBridge(
+        input=bridged_input,
+        tools=(*eager, *_regrouped(_first_occurrences(loaded, taken))),
+        bridged=True,
+    )

litellm/responses/litellm_completion_transformation/count_request.py

@@ -20,7 +20,7 @@ from litellm.responses.litellm_completion_transformation.transformation import (
     LiteLLMCompletionResponsesConfig,
 )
 from litellm.responses.main import previous_response_client_ids
-from litellm.types.llms.openai import ResponseInputParam
+from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam
 from litellm.utils import ProviderConfigManager
 
 _COUNTED_CHAT_PARAMS: Final = (
@@ -119,14 +119,13 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> M
     if not isinstance(responses_input, list):
         return responses_request
     input_items: Final = cast(ResponseInputParam, responses_input)  # cast-ok: forwarded as sent, as inference does
-    hoisted: Final = hoist_additional_tools(input_items, None)
+    top_level_tools: Final = cast(  # cast-ok: tools are forwarded as sent, as inference does
+        "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(responses_request.get("tools") or ())
+    )
+    hoisted: Final = hoist_additional_tools(input_items, top_level_tools)
     if not hoisted.hoisted:
         return responses_request
-    return {
-        **responses_request,
-        "input": hoisted.input,
-        "tools": [*_ITEMS.validate_python(responses_request.get("tools") or ()), *hoisted.hoisted],
-    }
+    return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)}
 
 
 def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]:

litellm/responses/litellm_completion_transformation/handler.py

@@ -2,17 +2,21 @@
 Handler for transforming responses api requests to litellm.completion requests
 """
 
-from collections.abc import Coroutine, Mapping
+from collections.abc import Coroutine, Mapping, Sequence
+from dataclasses import dataclass
 from typing import Final
 
 from pydantic import TypeAdapter
 
 import litellm
-from litellm.responses.additional_tools import hoist_additional_tools
+from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
+from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search
 from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary
+from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, StoredTurns
 from litellm.responses.litellm_completion_transformation.streaming_iterator import (
     LiteLLMCompletionStreamingIterator,
 )
+from litellm.responses.litellm_completion_transformation.tool_search import has_client_tool_search
 from litellm.responses.litellm_completion_transformation.transformation import (
     LiteLLMCompletionResponsesConfig,
     with_compaction_items_as_params,
@@ -28,6 +32,28 @@ from litellm.types.utils import ModelResponse
 _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
 
 
+@dataclass(frozen=True, slots=True)
+class _BridgedRequest:
+    input: str | ResponseInputParam
+    responses_api_request: ResponsesAPIOptionalRequestParams
+
+
+def _bridged_request(
+    hoisted: HoistedAdditionalTools,
+    responses_api_request: ResponsesAPIOptionalRequestParams,
+    loaded_earlier: Sequence[Mapping[str, object]],
+) -> _BridgedRequest:
+    searched: Final = bridge_client_tool_search(hoisted.input, hoisted.tools, loaded_earlier)
+    return _BridgedRequest(
+        input=searched.input,
+        responses_api_request=(
+            {**responses_api_request, "tools": list(searched.tools)}
+            if hoisted.hoisted or searched.bridged
+            else responses_api_request
+        ),
+    )
+
+
 class LiteLLMCompletionTransformationHandler:
     def response_api_handler(
         self,
@@ -48,10 +74,22 @@ class LiteLLMCompletionTransformationHandler:
         | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
     ):
         hoisted: Final = hoist_additional_tools(input, responses_api_request.get("tools"))
-        bridged_input: Final = hoisted.input
-        bridged_request: Final[ResponsesAPIOptionalRequestParams] = (
-            {**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request
-        )
+        previous_response_id: Final = responses_api_request.get("previous_response_id")
+        if _is_async and previous_response_id and has_client_tool_search(hoisted.tools):
+            return self._async_response_api_handler_after_earlier_searches(
+                previous_response_client_ids,
+                previous_response_id,
+                hoisted,
+                model=model,
+                responses_api_request=responses_api_request,
+                custom_llm_provider=custom_llm_provider,
+                stream=stream,
+                extra_headers=extra_headers,
+                request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
+            )
+        bridged: Final = _bridged_request(hoisted, responses_api_request, ())
+        bridged_input: Final = bridged.input
+        bridged_request: Final = bridged.responses_api_request
         litellm_completion_request: Final[dict] = (
             LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
                 model=model,
@@ -73,13 +111,8 @@ class LiteLLMCompletionTransformationHandler:
                 **kwargs,
             )
 
-        completion_args: Final = {}
-        completion_args.update(kwargs)
-        completion_args.update(litellm_completion_request)
-        completion_args["_skip_responses_api_bridge"] = True
-
         litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion(
-            **completion_args,
+            **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
         )
 
         if isinstance(litellm_completion_response, ModelResponse):
@@ -104,9 +137,53 @@ class LiteLLMCompletionTransformationHandler:
             )
         raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
 
+    async def _async_response_api_handler_after_earlier_searches(
+        self,
+        previous_response_client_ids: tuple[str, ...],
+        previous_response_id: str,
+        hoisted: HoistedAdditionalTools,
+        /,
+        *,
+        model: str,
+        responses_api_request: ResponsesAPIOptionalRequestParams,
+        custom_llm_provider: str | None,
+        stream: bool | None,
+        extra_headers: Mapping[str, object] | None,
+        request_kwargs: Mapping[str, object],
+    ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
+        stored_turns: Final = await ResponsesSessionHandler.get_stored_turns_for_previous_response_id(
+            previous_response_id, previous_response_client_ids
+        )
+        bridged: Final = _bridged_request(
+            hoisted,
+            responses_api_request,
+            ResponsesSessionHandler.client_tool_search_tools_loaded_before(
+                stored_turns, previous_response_id, previous_response_client_ids
+            ),
+        )
+        return await self.async_response_api_handler(
+            previous_response_client_ids,
+            stored_turns,
+            litellm_completion_request=(
+                LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
+                    model=model,
+                    input=bridged.input,
+                    responses_api_request=bridged.responses_api_request,
+                    custom_llm_provider=custom_llm_provider,
+                    stream=stream,
+                    extra_headers=extra_headers,
+                    **request_kwargs,
+                )
+            ),
+            request_input=bridged.input,
+            responses_api_request=bridged.responses_api_request,
+            **request_kwargs,
+        )
+
     async def async_response_api_handler(
         self,
         previous_response_client_ids: tuple[str, ...] = (),
+        stored_turns: StoredTurns | None = None,
         /,
         *,
         litellm_completion_request: dict,
@@ -122,16 +199,12 @@ class LiteLLMCompletionTransformationHandler:
                 instructions=responses_api_request.get("instructions"),
                 request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
                 previous_response_client_ids=previous_response_client_ids,
+                stored_turns=stored_turns,
                 input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)),
             )
 
-        acompletion_args: Final = {}
-        acompletion_args.update(kwargs)
-        acompletion_args.update(litellm_completion_request)
-        acompletion_args["_skip_responses_api_bridge"] = True
-
         litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion(
-            **acompletion_args,
+            **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
         )
         if isinstance(litellm_completion_response, ModelResponse):
             responses_api_response: Final[ResponsesAPIResponse] = (

litellm/responses/litellm_completion_transformation/session_handler.py

@@ -9,6 +9,10 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
 from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler
+from litellm.responses.litellm_completion_transformation.client_tool_search import (
+    bridge_client_tool_search,
+    client_tool_search_loaded_tools,
+)
 from litellm.responses.litellm_completion_transformation.compaction_items import (
     holds_compaction_summary,
     input_items,
@@ -88,6 +92,12 @@ class _ReplayStart:
     at_output: bool
 
 
+@dataclass(frozen=True, slots=True)
+class StoredTurns:
+    spend_logs: tuple["SpendLogsPayload", ...]
+    proxy_server_requests: tuple[Mapping[str, object] | None, ...]
+
+
 def _compaction_point(
     index: int, spend_log: "SpendLogsPayload", proxy_server_request: Mapping[str, object] | None
 ) -> _ReplayStart | None:
@@ -190,6 +200,7 @@ class ResponsesSessionHandler:
         previous_response_id: str,
         item_cache_control: bool = False,
         client_response_ids: tuple[str, ...] = (),
+        stored_turns: StoredTurns | None = None,
     ) -> ChatCompletionSession:
         """
         Return the chat completion message history for a previous response id
@@ -200,23 +211,21 @@ class ResponsesSessionHandler:
         )
 
         verbose_proxy_logger.debug("inside get_chat_completion_message_history_for_previous_response_id")
-        all_spend_logs: Final = [
-            spend_log
-            for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(
+        turns: Final = (
+            stored_turns
+            if stored_turns is not None
+            else await ResponsesSessionHandler.get_stored_turns_for_previous_response_id(
                 previous_response_id, client_response_ids
             )
-            if _is_a_conversation_turn(spend_log)
-        ]
+        )
+        all_spend_logs: Final = turns.spend_logs
         verbose_proxy_logger.debug("found %s spend logs for this response id", len(all_spend_logs))
 
         litellm_session_id: str | None = None
         if len(all_spend_logs) > 0:
             litellm_session_id = all_spend_logs[0].get("session_id")
 
-        proxy_server_requests: Final = [
-            await ResponsesSessionHandler.get_proxy_server_request_from_spend_log(spend_log=spend_log)
-            for spend_log in all_spend_logs
-        ]
+        proxy_server_requests: Final = turns.proxy_server_requests
         replayable_turns: Final = _turns_through_referenced_response(
             all_spend_logs, _referenced_request_ids(previous_response_id, client_response_ids)
         )
@@ -252,6 +261,45 @@ class ResponsesSessionHandler:
             instructions=_latest_replayable_instructions(proxy_server_requests),
         )
 
+    @staticmethod
+    async def get_stored_turns_for_previous_response_id(
+        previous_response_id: str,
+        client_response_ids: tuple[str, ...] = (),
+    ) -> StoredTurns:
+        spend_logs: Final = tuple(
+            spend_log
+            for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(
+                previous_response_id, client_response_ids
+            )
+            if _is_a_conversation_turn(spend_log)
+        )
+        return StoredTurns(
+            spend_logs=spend_logs,
+            proxy_server_requests=tuple(
+                [
+                    await ResponsesSessionHandler.get_proxy_server_request_from_spend_log(spend_log=spend_log)
+                    for spend_log in spend_logs
+                ]
+            ),
+        )
+
+    @staticmethod
+    def client_tool_search_tools_loaded_before(
+        stored_turns: StoredTurns,
+        previous_response_id: str,
+        client_response_ids: tuple[str, ...] = (),
+    ) -> tuple[Mapping[str, object], ...]:
+        turns: Final = _turns_through_referenced_response(
+            stored_turns.spend_logs, _referenced_request_ids(previous_response_id, client_response_ids)
+        )
+        return tuple(
+            chain.from_iterable(
+                client_tool_search_loaded_tools(request.get("input"))
+                for request in stored_turns.proxy_server_requests[:turns]
+                if request is not None
+            )
+        )
+
     @staticmethod
     def _history_of_turns(
         spend_logs: Sequence["SpendLogsPayload"],
@@ -310,7 +358,7 @@ class ResponsesSessionHandler:
         if response_input_param:
             chat_completion_message_history.extend(
                 LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
-                    input=response_input_param,
+                    input=bridge_client_tool_search(response_input_param, None).input,
                     responses_api_request={},
                     replay_reasoning=True,
                     item_cache_control=item_cache_control,
@@ -325,7 +373,7 @@ class ResponsesSessionHandler:
         if (responses_api_output := _stored_responses_api_output(_response_output)) is not None:
             chat_completion_message_history.extend(
                 LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
-                    input=responses_api_output,
+                    input=bridge_client_tool_search(responses_api_output, None).input,
                     responses_api_request={},
                     replay_reasoning=True,
                     item_cache_control=item_cache_control,

litellm/responses/litellm_completion_transformation/streaming_iterator.py

@@ -1,7 +1,8 @@
 import time
 import uuid
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from itertools import filterfalse
+from types import MappingProxyType
 from typing import Any, Final, cast
 
 import litellm
@@ -18,9 +19,9 @@ from litellm.responses.litellm_completion_transformation.custom_tools import (
 )
 from litellm.responses.litellm_completion_transformation.reasoning_items import mint_reasoning_item_id
 from litellm.responses.litellm_completion_transformation.tool_search import (
+    ToolSearchExecution,
     build_tool_search_call,
-    has_server_tool_search,
-    is_anthropic_tool_search_use,
+    tool_search_routing,
 )
 from litellm.responses.litellm_completion_transformation.transformation import (
     LiteLLMCompletionResponsesConfig,
@@ -30,6 +31,7 @@ from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.openai import (
     PART_UNION_TYPES,
     BaseLiteLLMOpenAIResponseObject,
+    ChatCompletionAnnotation,
     ContentPartAddedEvent,
     ContentPartDoneEvent,
     ContentPartDonePartOutputText,
@@ -56,6 +58,7 @@ from litellm.types.llms.openai import (
     WebSearchCallInProgressEvent,
     WebSearchCallSearchingEvent,
 )
+from litellm.types.responses.main import GenericResponseOutputItemContentAnnotation
 from litellm.types.utils import Delta as ChatCompletionDelta
 from litellm.types.utils import (
     ModelResponse,
@@ -105,7 +108,6 @@ def _delta_carries_output(delta: ChatCompletionDelta) -> bool:
         getattr(delta, "reasoning_content", None),
         delta.tool_calls,
         delta.function_call,
-        getattr(delta, "annotations", None),
         getattr(delta, "images", None),
         getattr(delta, "audio", None),
     )
@@ -150,7 +152,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         self.sent_output_text_done_event: bool = False
         self.sent_output_content_part_done_event: bool = False
         self.sent_output_item_done_event: bool = False
-        self.sent_annotation_events: bool = False
+        self._annotation_index: int = 0
+        self._annotations_awaiting_message: tuple[ChatCompletionAnnotation, ...] = ()
         self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None
         self.completed_response = None
         self.final_text: str = ""
@@ -190,16 +193,21 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         self._namespace_tool_names = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(request_tools)
         self._web_search_calls: dict[str, object] = {}  # mutable-ok: latest call by provider id
         self._queued_web_search_call_ids: set[str] = set()  # mutable-ok: emitted call ids
-        self._tool_search_requested: bool = has_server_tool_search(request_tools)
-        self._tool_search_call_ids: frozenset[str] = frozenset()
+        self._tool_search = tool_search_routing(request_tools)
+        self._tool_search_executions: Mapping[str, ToolSearchExecution] = MappingProxyType({})
 
-    def _is_tool_search_call(self, call_id: str, fn_name: str) -> bool:
-        if call_id in self._tool_search_call_ids:
-            return True
-        if not self._tool_search_requested or not is_anthropic_tool_search_use(call_id, fn_name):
-            return False
-        self._tool_search_call_ids = self._tool_search_call_ids | {call_id}
-        return True
+    async def aclose(self) -> None:
+        await self.litellm_custom_stream_wrapper.aclose()
+
+    def _tool_search_execution(self, call_id: str, fn_name: str) -> ToolSearchExecution | None:
+        known: Final = self._tool_search_executions.get(call_id)
+        if known is not None:
+            return known
+        execution: Final = self._tool_search.execution_of(call_id, fn_name)
+        if execution is None:
+            return None
+        self._tool_search_executions = MappingProxyType({**self._tool_search_executions, call_id: execution})
+        return execution
 
     def _next_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> OutputItemAddedEvent:
         self._sequence_number += 1
@@ -214,18 +222,22 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
     def _queue_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> None:
         self._pending_tool_events.append(self._next_output_item_added(output_index, item))
 
-    def _queue_tool_search_call_added(self, call_id: str, output_index: int, args_delta: str) -> None:
+    def _queue_tool_search_call_added(
+        self, call_id: str, output_index: int, args_delta: str, execution: ToolSearchExecution
+    ) -> None:
         if call_id not in self._tool_args_by_call_id:
             self._tool_args_by_call_id[call_id] = ""
-            item: Final = build_tool_search_call(call_id, "", "in_progress")
+            item: Final = build_tool_search_call(call_id, "", "in_progress", execution)
             self._tool_item_id_by_call_id[call_id] = item.id
             self._queue_output_item_added(output_index, BaseLiteLLMOpenAIResponseObject(**item.model_dump()))
         self._tool_args_by_call_id[call_id] += args_delta
 
-    def _queue_tool_search_call_done(self, call_id: str, output_index: int, final_args: str) -> None:
-        self._queue_tool_search_call_added(call_id, output_index, "")
+    def _queue_tool_search_call_done(
+        self, call_id: str, output_index: int, final_args: str, execution: ToolSearchExecution
+    ) -> None:
+        self._queue_tool_search_call_added(call_id, output_index, "", execution)
         item: Final = build_tool_search_call(
-            call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed"
+            call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution
         )
         self._sequence_number += 1
         self._pending_tool_events.append(
@@ -339,7 +351,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         namespace_kwargs: Final = {"namespace": tool_namespace} if tool_namespace else {}
         return {**item_kwargs, "name": tool_name, **namespace_kwargs}
 
-    def _is_reasoning_end(self, chunk):
+    def _is_reasoning_end(self, chunk: ModelResponseStream) -> bool:
         if not chunk.choices:
             return False
         delta: Final = chunk.choices[0].delta
@@ -350,7 +362,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         if hasattr(delta, "thinking_blocks") and delta.thinking_blocks:
             return False
 
-        return delta.content or delta.function_call or delta.tool_calls or chunk.choices[0].finish_reason is not None
+        return bool(
+            delta.content or delta.function_call or delta.tool_calls or chunk.choices[0].finish_reason is not None
+        )
 
     def _reserve_web_search_indexes(self, provider_fields: object) -> None:
         if not isinstance(provider_fields, dict):
@@ -428,10 +442,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             else:
                 fn_name = str(getattr(fn, "name", "") or "")
                 fn_args_delta = serialize_tool_call_arguments(getattr(fn, "arguments", ""))
+            if call_id not in self._tool_output_index_by_call_id:
+                self._queue_held_annotations()
             output_index = self._get_or_assign_tool_output_index(call_id)
 
-            if self._is_tool_search_call(call_id, fn_name):
-                self._queue_tool_search_call_added(call_id, output_index, fn_args_delta)
+            if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None:
+                self._queue_tool_search_call_added(call_id, output_index, fn_args_delta, search_execution)
                 continue
             self._started_tool_call = True
 
@@ -499,8 +515,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
                     self._queue_web_search_events(call_id, web_search_call)
                     self._queued_web_search_call_ids.add(call_id)
                 continue
-            if self._is_tool_search_call(call_id, fn_name):
-                self._queue_tool_search_call_done(call_id, output_index, fn_args)
+            if (search_execution := self._tool_search_execution(call_id, fn_name)) is not None:
+                self._queue_tool_search_call_done(call_id, output_index, fn_args, search_execution)
                 continue
 
             # Track if this is a new tool call that wasn't streamed
@@ -782,8 +798,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         return event
 
     def _queue_message_item_added_events(self) -> None:
-        if self._cached_item_id is None:
-            self._cached_item_id = f"msg_{uuid.uuid4()}"
+        item_id: Final = self._message_item_id()
         self.sent_message_item_added_event = True
         self.sent_content_part_added_event = True
         self._sequence_number += 1
@@ -792,7 +807,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             output_index=self._message_index(),
             item=BaseLiteLLMOpenAIResponseObject(
                 **{
-                    "id": self._cached_item_id,
+                    "id": item_id,
                     "type": "message",
                     "role": "assistant",
                     "status": "in_progress",
@@ -1081,6 +1096,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         if self.litellm_model_response:
             self._queue_compaction_item_added()
             self._queue_compaction_item_done()
+            self._queue_held_annotations()
             if self._pending_response_events:
                 return self._pending_response_events.pop(0)
             # If tool calls exist, emit tool events before finishing/response.completed.
@@ -1163,7 +1179,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         delta: Final = chunk.choices[0].delta if chunk.choices else None
         if delta and getattr(delta, "reasoning_content", None):
             self._accumulated_reasoning_content_parts.append(delta.reasoning_content)
-        if not self._is_reasoning_end(chunk):
+        if self._is_reasoning_end(chunk):
+            self._queue_reasoning_done_events()
+
+    def _queue_reasoning_done_events(self) -> None:
+        if not self._reasoning_active or self._reasoning_done_emitted:
             return
         reasoning_content: Final = "".join(self._accumulated_reasoning_content_parts)
         self._cached_reasoning_item_id = (
@@ -1237,9 +1257,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
                             self._snapshot_chunk_for_stream_chunk_builder(chunk)
                         )
                         self._queue_reasoning_done_events_on_reasoning_end(chunk)
-                        response_api_chunk = self._transform_chat_completion_chunk_to_response_api_chunk(chunk)
-                        if response_api_chunk:
-                            self._pending_response_events.append(response_api_chunk)
+                        self._queue_chunk_events(chunk)
 
                     if self._pending_response_events:
                         return self._pending_response_events.pop(0)
@@ -1300,9 +1318,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
                         self._snapshot_chunk_for_stream_chunk_builder(cast(ModelResponseStream, chunk))
                     )
                     self._queue_reasoning_done_events_on_reasoning_end(chunk)
-                    response_api_chunk = self._transform_chat_completion_chunk_to_response_api_chunk(chunk)
-                    if response_api_chunk:
-                        self._pending_response_events.append(response_api_chunk)
+                    self._queue_chunk_events(chunk)
                     if self._pending_response_events:
                         return self._pending_response_events.pop(0)
                     # Otherwise, loop to next chunk
@@ -1313,6 +1329,58 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             self.finished = True
             raise e
 
+    def _queue_chunk_events(self, chunk: ModelResponseStream) -> None:
+        self._hold_chunk_annotations(chunk)
+        response_api_chunk: Final = self._transform_chat_completion_chunk_to_response_api_chunk(chunk)
+        if response_api_chunk:
+            self._pending_response_events.append(response_api_chunk)
+        if self.sent_message_item_added_event:
+            self._queue_held_annotations()
+
+    def _hold_chunk_annotations(self, chunk: ModelResponseStream) -> None:
+        chunk_annotations: Final = (getattr(chunk.choices[0].delta, "annotations", None) or ()) if chunk.choices else ()
+        self._annotations_awaiting_message = (*self._annotations_awaiting_message, *chunk_annotations)
+
+    def _queue_held_annotations(self) -> None:
+        annotations: Final = self._annotations_awaiting_message
+        if not annotations:
+            return
+        self._annotations_awaiting_message = ()
+        self._queue_reasoning_done_events()
+        self._queue_annotation_added_events(annotations)
+
+    def _queue_annotation_added_events(self, annotations: Sequence[ChatCompletionAnnotation]) -> None:
+        if not self.sent_message_item_added_event:
+            self._queue_message_item_added_events()
+        item_id: Final = self._message_item_id()
+        response_annotations: Final = (
+            LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations(
+                annotations=annotations
+            )
+        )
+        self._pending_response_events.extend(
+            self._annotation_added_event(item_id, annotation) for annotation in response_annotations
+        )
+
+    def _message_item_id(self) -> str:
+        if self._cached_item_id is None:
+            self._cached_item_id = f"msg_{uuid.uuid4()}"
+        return self._cached_item_id
+
+    def _annotation_added_event(
+        self, item_id: str, annotation: GenericResponseOutputItemContentAnnotation
+    ) -> OutputTextAnnotationAddedEvent:
+        event: Final = OutputTextAnnotationAddedEvent(
+            type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED,
+            item_id=item_id,
+            output_index=self._message_index(),
+            content_index=0,
+            annotation_index=self._annotation_index,
+            annotation=annotation.model_dump(),
+        )
+        self._annotation_index += 1
+        return event
+
     def _transform_chat_completion_chunk_to_response_api_chunk(
         self, chunk: ModelResponseStream
     ) -> ResponsesAPIStreamingResponse | None:
@@ -1321,37 +1389,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
 
         This currently handles emitting the OutputTextDeltaEvent, which is used by other tools using the responses API
         and the ReasoningSummaryTextDeltaEvent, which is used by the responses API to emit reasoning content.
-        It also handles emitting annotation.added events when annotations are detected in the chunk.
         """
-        if self._cached_item_id is None:
-            self._cached_item_id = f"msg_{uuid.uuid4()}"
-        item_id: Final = self._cached_item_id
+        item_id: Final = self._message_item_id()
 
-        # Check if this chunk has annotations first (before processing text/reasoning)
-        # This ensures we detect and queue annotation events from the annotation chunk
-        if chunk.choices and hasattr(chunk.choices[0].delta, "annotations"):
-            annotations: Final = chunk.choices[0].delta.annotations
-            if annotations and self.sent_annotation_events is False:
-                self.sent_annotation_events = True
-                # Store annotation events to emit them one by one
-                if not hasattr(self, "_pending_annotation_events"):
-                    response_annotations = LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations(
-                        annotations=annotations
-                    )
-                    self._pending_annotation_events = []
-                    for idx, annotation in enumerate(response_annotations):
-                        annotation_dict = (
-                            annotation.model_dump() if hasattr(annotation, "model_dump") else dict(annotation)
-                        )
-                        event = OutputTextAnnotationAddedEvent(
-                            type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED,
-                            item_id=item_id,
-                            output_index=self._message_index(),
-                            content_index=0,
-                            annotation_index=idx,
-                            annotation=annotation_dict,
-                        )
-                        self._pending_annotation_events.append(event)
         # Priority 1: Handle reasoning content (highest priority)
         if (
             chunk.choices
@@ -1395,13 +1435,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             if self._pending_tool_events:
                 return self._pending_tool_events.pop(0)
 
-        # Priority 4: If we have pending annotation events, emit the next one
-        # This happens when the current chunk has no text/reasoning content
-        if hasattr(self, "_pending_annotation_events") and self._pending_annotation_events:
-            event = self._pending_annotation_events.pop(0)
-            return event
-
-        # Priority 5: If we have pending tool events (from earlier chunk), emit the next one
         if self._pending_tool_events:
             return self._pending_tool_events.pop(0)
 
@@ -1444,7 +1477,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
     def _is_unstreamed_empty_message(self, item: object) -> bool:
         if getattr(item, "type", None) != "message" or self._message_output_index is not None:
             return False
-        return not any(getattr(part, "text", None) for part in attribute_items(item, "content"))
+        return not any(
+            getattr(part, "text", None) or getattr(part, "annotations", None)
+            for part in attribute_items(item, "content")
+        )
 
     def finished_round_event(self) -> ResponseCompletedEvent | ResponseIncompleteEvent | None:
         if not isinstance(self.litellm_model_response, ModelResponse):

litellm/responses/litellm_completion_transformation/tool_search.py

@@ -1,4 +1,5 @@
 from collections.abc import Iterable, Iterator, Mapping, Sequence, Set
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Final, Literal, TypeAlias
 
 from pydantic import ConfigDict, TypeAdapter, ValidationError
@@ -18,25 +19,31 @@ if TYPE_CHECKING:
 __all__ = [
     "ANTHROPIC_TOOL_SEARCH_TOOL_NAME",
     "ANTHROPIC_TOOL_SEARCH_TOOL_TYPE",
+    "CLIENT_TOOL_SEARCH_FUNCTION_NAME",
     "CompletionRoute",
     "ToolSearchCallStatus",
+    "ToolSearchExecution",
+    "ToolSearchRouting",
     "answers_tool_search",
     "anthropic_tool_search_chat_tool",
     "build_tool_search_call",
     "chat_config_builds_anthropic_request",
     "chat_messages_without_tool_search_calls",
     "chat_route_rejects_unanswered_tool_calls",
+    "has_client_tool_search",
     "has_server_tool_search",
     "holds_tool_search_use",
     "input_without_unanswered_anthropic_tool_search_calls",
     "is_anthropic_tool_search_chat_tool",
     "is_anthropic_tool_search_use",
+    "is_client_tool_search_tool",
     "is_openai_tool_search_tool",
     "is_server_tool_search_tool",
     "provider_builds_anthropic_request",
     "provider_chat_config",
     "runs_tool_search_on_the_server",
     "tool_search_call_ids",
+    "tool_search_routing",
     "tools_with_anthropic_tool_search_mapped",
     "tools_without_anthropic_tool_search",
     "unanswered_tool_search_call_ids",
@@ -46,7 +53,9 @@ __all__ = [
 
 ANTHROPIC_TOOL_SEARCH_TOOL_TYPE: Final = "tool_search_tool_regex_20251119"
 ANTHROPIC_TOOL_SEARCH_TOOL_NAME: Final = "tool_search_tool_regex"
+CLIENT_TOOL_SEARCH_FUNCTION_NAME: Final = "tool_search"
 ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"]
+ToolSearchExecution: TypeAlias = Literal["server", "client"]
 
 _ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object])
 _MESSAGE_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
@@ -79,6 +88,35 @@ def has_server_tool_search(tools: Sequence[object] | None) -> bool:
     return any(is_server_tool_search_tool(tool) for tool in tools or ())
 
 
+def is_client_tool_search_tool(tool: object) -> bool:
+    try:
+        parsed: Final = _ToolSearchFields.model_validate(tool)
+    except ValidationError:
+        return False
+    return parsed.type == "tool_search" and parsed.execution == "client"
+
+
+def has_client_tool_search(tools: Sequence[object] | None) -> bool:
+    return any(is_client_tool_search_tool(tool) for tool in tools or ())
+
+
+@dataclass(frozen=True, slots=True)
+class ToolSearchRouting:
+    server: bool
+    client: bool
+
+    def execution_of(self, tool_id: str, tool_name: str) -> ToolSearchExecution | None:
+        if self.server and is_anthropic_tool_search_use(tool_id, tool_name):
+            return "server"
+        if self.client and tool_name == CLIENT_TOOL_SEARCH_FUNCTION_NAME:
+            return "client"
+        return None
+
+
+def tool_search_routing(tools: Sequence[object] | None) -> ToolSearchRouting:
+    return ToolSearchRouting(server=has_server_tool_search(tools), client=has_client_tool_search(tools))
+
+
 def anthropic_tool_search_chat_tool() -> AnthropicToolSearchToolRegex:
     return AnthropicToolSearchToolRegex(type=ANTHROPIC_TOOL_SEARCH_TOOL_TYPE, name=ANTHROPIC_TOOL_SEARCH_TOOL_NAME)
 
@@ -346,13 +384,18 @@ def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None:
         return None
 
 
-def build_tool_search_call(tool_id: str, arguments_json: str, status: ToolSearchCallStatus) -> OutputToolSearchCall:
+def build_tool_search_call(
+    tool_id: str,
+    arguments_json: str,
+    status: ToolSearchCallStatus,
+    execution: ToolSearchExecution = "server",
+) -> OutputToolSearchCall:
     arguments: Final = _parsed_arguments(arguments_json)
     return OutputToolSearchCall(
         id=openai_shaped_tool_call_item_id("tool_search_call", tool_id),
         call_id=tool_id,
         arguments={} if arguments is None else arguments,
-        execution="server",
+        execution=execution,
         status="incomplete" if arguments is None else status,
         type="tool_search_call",
     )