Merge remote-tracking branch 'upstream/main' into fix-messages-header-image-cost

# Conflicts:
#	tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py

.circleci/scripts/run_integration.sh

@@ -128,6 +128,8 @@ fi
 sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
 awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
 
+export UV_OFFLINE=true
+
 if [ "$mode" = parity ]; then
   export INTEGRATION_WORKERS=1
 else

.github/workflows/compat-matrix-image.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - tests/e2e/claude_code/cron_vm/**
-      - tests/e2e/claude_code/pr_gate_version_resolver.py
+      - tests/e2e/cli_harness/npm_version_resolver.py
       - .github/workflows/compat-matrix-image.yml
   workflow_dispatch:
 
@@ -36,7 +36,7 @@ jobs:
             whoami
             gh --version
             uv --version
-            version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/claude_code/pr_gate_version_resolver.py)"
+            version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
             /opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli
             /tmp/claude-cli/claude --version
           '

.github/workflows/deploy-dev-on-main.yml

@@ -1,30 +0,0 @@
-name: Deploy main to gateway-dev
-
-on:
-  push:
-    branches: [main]
-
-permissions: {}
-
-concurrency:
-  group: deploy-dev-dispatch
-  cancel-in-progress: true
-
-jobs:
-  dispatch:
-    if: github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2
-        id: token
-        with:
-          app-id: ${{ secrets.LITELLM_OPS_DISPATCH_APP_ID }}
-          private-key: ${{ secrets.LITELLM_OPS_DISPATCH_APP_PRIVATE_KEY }}
-          owner: BerriAI
-          repositories: litellm-ops
-          permission-actions: write
-      - name: Dispatch the litellm-ops dev deploy
-        env:
-          GH_TOKEN: ${{ steps.token.outputs.token }}
-        run: gh api --method POST repos/BerriAI/litellm-ops/actions/workflows/deploy-dev.yml/dispatches -f ref=main

.github/workflows/test-linting.yml

@@ -208,7 +208,7 @@ jobs:
             exit 0
           fi
           retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
-          CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
+          CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
           tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
           PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
           uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \

.github/workflows/weekly_load_anomaly.yml

@@ -1,85 +0,0 @@
-name: "Weekly Load Anomaly Check"
-
-on:
-  schedule:
-    - cron: "0 12 * * 6"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  weekly-load-anomaly:
-    if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-    services:
-      postgres:
-        image: postgres:16.6
-        env:
-          POSTGRES_USER: llmproxy
-          POSTGRES_PASSWORD: dbpassword9090
-          POSTGRES_DB: litellm
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd "pg_isready -U llmproxy"
-          --health-interval 5s
-          --health-timeout 5s
-          --health-retries 10
-    env:
-      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
-      LITELLM_MASTER_KEY: sk-weekly-anomaly-check
-      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-      AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Cache the Rust build
-        uses: ./.github/actions/cache-cargo-build
-
-      - name: Install dependencies
-        run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
-
-      - name: Cache Prisma binaries
-        uses: ./.github/actions/cache-prisma-binaries
-
-      - name: Generate Prisma client
-        run: |
-          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
-
-      - name: Start the proxy
-        run: |
-          nohup uv run --no-sync litellm --config tests/e2e/perf/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
-          for _ in $(seq 1 90); do
-            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
-              exit 0
-            fi
-            sleep 2
-          done
-          echo "proxy never became live"
-          tail -n 100 proxy.log
-          exit 1
-
-      - name: Run the weekly session anomaly test
-        env:
-          E2E_WEEKLY_ANOMALY: "1"
-        run: |
-          uv run --no-sync pytest tests/e2e/perf/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
-
-      - name: Show proxy log on failure
-        if: failure()
-        run: tail -n 300 proxy.log

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -3,9 +3,11 @@
 
 import asyncio
 import base64
+import copy
 import json
 import time
 from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 from itertools import chain
 from types import MappingProxyType
@@ -34,7 +36,7 @@ from typing_extensions import ReadOnly, Unpack
 
 import litellm
 from litellm import Router, verbose_logger
-from litellm._internal_context import with_service_target
+from litellm._internal_context import internal_sub_call, post_response_phase, with_service_target
 from litellm._uuid import uuid
 from litellm.caching.caching import DualCache
 from litellm.constants import (
@@ -95,6 +97,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
     request_tags_from_metadata,
 )
 from litellm.repositories.managed_file_repository import ManagedFileRepository
+from litellm.responses.streaming_iterator import spawn_logging_task
 from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.openai import (  # pyright: ignore[reportAttributeAccessIssue]
     AllMessageValues,
@@ -395,6 +398,134 @@ class _RouterFileCallKwargs(TypedDict, total=False):
     custom_llm_provider: ReadOnly[str | None]
 
 
+_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
+_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"})
+_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"})
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderUpload:
+    model_id: str
+    provider_file_id: str
+    upload_call_id: str | None = None
+
+
+def _provider_uploads(file_object: OpenAIFileObject) -> tuple[_ProviderUpload, ...]:
+    file_hidden_params: Final = cast(  # cast-ok: preserve mapping operations on dynamic file metadata
+        dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
+    )
+    model_file_id_mapping: Final = _MODEL_FILE_ID_MAPPING.validate_python(
+        file_hidden_params.get("model_file_id_mapping") or {}
+    )
+    upload_call_id: Final = file_hidden_params.get("litellm_call_id")
+    return tuple(
+        _ProviderUpload(
+            model_id=model_id,
+            provider_file_id=provider_file_id,
+            upload_call_id=upload_call_id if isinstance(upload_call_id, str) else None,
+        )
+        for model_id, provider_file_id in model_file_id_mapping.items()
+    )
+
+
+def _keeps_one_upload_per_target(llm_router: Router, model_id: str) -> bool:
+    credentials: Final = _provider_file_retrieve_credentials(llm_router=llm_router, model_id=model_id)
+    return (credentials or {}).get("custom_llm_provider") == LlmProviders.LITELLM_PROXY.value
+
+
+def _is_kept_upload(uploads: Sequence[_ProviderUpload], index: int, per_target_model_ids: frozenset[str]) -> bool:
+    upload: Final = uploads[index]
+    other_uploads: Final = uploads[index + 1 :] if upload.model_id in per_target_model_ids else uploads[:index]
+    return all(other.model_id != upload.model_id for other in other_uploads)
+
+
+def _upload_from_trailing_id_fields(decoded_unified_file_id: str) -> _ProviderUpload:
+    trailing_fields: Final = decoded_unified_file_id.rpartition(";llm_output_file_id,")[2]
+    provider_file_id, _, model_id = trailing_fields.partition(";llm_output_file_model_id,")
+    return _ProviderUpload(model_id=model_id, provider_file_id=provider_file_id)
+
+
+def _carried_upload_missing_from(unified_file_id: str, model_file_ids: Mapping[str, str]) -> _ProviderUpload | None:
+    decoded: Final = _is_base64_encoded_unified_file_id(unified_file_id)
+    carried: Final = _upload_from_trailing_id_fields(decoded) if decoded else None
+    if carried is None or model_file_ids.get(carried.model_id) in (None, carried.provider_file_id):
+        return None
+    return carried
+
+
+async def _delete_provider_file(
+    llm_router: Router,
+    model_id: str,
+    provider_file_id: str,
+    request_data: Mapping[str, object],
+) -> None:
+    credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
+    delete_data: Final = {
+        **{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS},
+        **(
+            {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
+            if credentials is not None
+            else {}
+        ),
+    }
+    await llm_router.afile_delete(
+        model=model_id,
+        file_id=provider_file_id,
+        **cast(_RouterFileCallKwargs, delete_data),
+    )
+
+
+def _with_own_nested_dicts(request_data: Mapping[str, object]) -> MappingProxyType[str, object]:
+    return MappingProxyType({k: copy.copy(v) if isinstance(v, dict) else v for k, v in request_data.items()})
+
+
+async def _delete_redundant_upload(
+    llm_router: Router,
+    upload: _ProviderUpload,
+    delete_request: Mapping[str, object],
+    parent_call_id: str | None,
+) -> None:
+    try:
+        with internal_sub_call(parent_call_id):
+            await _delete_provider_file(
+                llm_router, upload.model_id, upload.provider_file_id, _with_own_nested_dicts(delete_request)
+            )
+    except Exception as error:
+        verbose_logger.warning(
+            f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} "
+            f"at deployment model_id={sanitize_for_log(upload.model_id)}: "
+            f"{type(error).__name__} {sanitize_for_log(error)}"
+        )
+
+
+async def _delete_redundant_uploads(
+    llm_router: Router,
+    uploads: Sequence[_ProviderUpload],
+    delete_request: Mapping[str, object],
+) -> None:
+    await asyncio.gather(
+        *(_delete_redundant_upload(llm_router, upload, delete_request, upload.upload_call_id) for upload in uploads)
+    )
+
+
+def _start_deleting_redundant_uploads(
+    llm_router: Router,
+    uploads: Sequence[_ProviderUpload],
+    create_file_request: CreateFileRequest,
+) -> None:
+    if not uploads:
+        return
+    delete_request: Final = MappingProxyType(
+        {k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS}
+    )
+    with post_response_phase():
+        spawn_logging_task(
+            asyncio.get_running_loop(),
+            _delete_redundant_uploads(llm_router, uploads, delete_request),
+            task_name="managed file repeat upload cleanup",
+        )
+
+
 def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
     return prisma_client.db
 
@@ -2034,16 +2165,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             target_model_names_list=target_model_names_list,
         )
 
-        ## STORE MODEL MAPPINGS IN DB
-        model_mappings: Dict[str, str] = {}
-
-        for file_object in responses:
-            file_hidden_params = cast(  # cast-ok: preserve mapping operations on dynamic file metadata
-                dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
-            )
-            model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
-            if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
-                model_mappings.update(model_file_id_mapping)
+        uploads: Final = tuple(chain.from_iterable(map(_provider_uploads, responses)))
+        per_target_model_ids: Final = frozenset(
+            model_id
+            for model_id in {upload.model_id for upload in uploads}
+            if _keeps_one_upload_per_target(llm_router, model_id)
+        )
+        kept_uploads: Final = tuple(
+            upload for index, upload in enumerate(uploads) if _is_kept_upload(uploads, index, per_target_model_ids)
+        )
+        model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads}
 
         await self.store_unified_file_id(
             file_id=response.id,
@@ -2052,6 +2183,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             model_mappings=model_mappings,
             user_api_key_dict=user_api_key_dict,
         )
+        _start_deleting_redundant_uploads(
+            llm_router,
+            tuple(
+                upload
+                for upload in uploads
+                if upload not in kept_uploads and upload.model_id not in per_target_model_ids
+            ),
+            create_file_request,
+        )
 
         # Emit Prometheus metrics for managed file creation
         prom_logger = self._get_prometheus_logger()
@@ -2692,23 +2832,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         specific_model_file_id_mapping: Final = model_file_id_mapping.get(file_id)
         if not specific_model_file_id_mapping:
             return
-        filtered_data: Final = {
-            k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
-        }
         for model_id, model_file_id in specific_model_file_id_mapping.items():
-            credentials = llm_router.registry().credentials(model_id, form="with_provider")
-            delete_data = {
-                **filtered_data,
-                **(
-                    {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))}
-                    if credentials is not None
-                    else {}
-                ),
-            }
-            await llm_router.afile_delete(
-                model=model_id,
-                file_id=model_file_id,
-                **cast(_RouterFileCallKwargs, delete_data),
+            await _delete_provider_file(llm_router, model_id, model_file_id, data)
+        unrecorded_upload: Final = _carried_upload_missing_from(file_id, specific_model_file_id_mapping)
+        delete_call_id: Final = data.get("litellm_call_id")
+        if unrecorded_upload is not None:
+            await _delete_redundant_upload(
+                llm_router, unrecorded_upload, data, delete_call_id if isinstance(delete_call_id, str) else None
             )
 
     async def afile_content(

litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs

@@ -124,12 +124,86 @@ pub enum AnthropicBeta {
     ToolSearchTool20251019,
     #[strum(serialize = "inline-tools-2026-09-15")]
     InlineTools20260915,
+    #[strum(serialize = "afk-mode-2026-01-31")]
+    AfkMode20260131,
+    #[strum(serialize = "agent-memory-2026-07-22")]
+    AgentMemory20260722,
+    #[strum(serialize = "analytics-usage-cost-2026-08-24")]
+    AnalyticsUsageCost20260824,
+    #[strum(serialize = "cache-diagnosis-2026-04-07")]
+    CacheDiagnosis20260407,
+    #[strum(serialize = "ce-plugins-2026-09-01")]
+    CePlugins20260901,
+    #[strum(serialize = "ce-user-management-2026-07-13")]
+    CeUserManagement20260713,
+    #[strum(serialize = "code-execution-2025-05-22")]
+    CodeExecution20250522,
+    #[strum(serialize = "computer-use-2024-10-22")]
+    ComputerUse20241022,
+    #[strum(serialize = "dev-full-thinking-2025-05-14")]
+    DevFullThinking20250514,
+    #[strum(serialize = "dreaming-2026-04-21")]
+    Dreaming20260421,
+    #[strum(serialize = "experimental-claude-tag-2026-10-21")]
+    ExperimentalClaudeTag20261021,
+    #[strum(serialize = "extended-cache-ttl-2025-04-11")]
+    ExtendedCacheTtl20250411,
+    #[strum(serialize = "fallback-credit-2026-06-01")]
+    FallbackCredit20260601,
+    #[strum(serialize = "fallback-credit-2026-07-01")]
+    FallbackCredit20260701,
+    #[strum(serialize = "managed-agents-2026-04-01")]
+    ManagedAgents20260401,
+    #[strum(serialize = "mcp-client-2026-09-15")]
+    McpClient20260915,
+    #[strum(serialize = "mcp-tunnels-2026-06-22")]
+    McpTunnels20260622,
+    #[strum(serialize = "message-batches-2024-09-24")]
+    MessageBatches20240924,
+    #[strum(serialize = "message-threads-2026-08-12")]
+    MessageThreads20260812,
+    #[strum(serialize = "mid-conversation-system-clear-at-2026-08-21")]
+    MidConversationSystemClearAt20260821,
+    #[strum(serialize = "model-context-window-exceeded-2025-08-26")]
+    ModelContextWindowExceeded20250826,
+    #[strum(serialize = "output-300k-2026-03-24")]
+    Output300k20260324,
+    #[strum(serialize = "pdfs-2024-09-25")]
+    Pdfs20240925,
+    #[strum(serialize = "prompt-caching-2024-07-31")]
+    PromptCaching20240731,
+    #[strum(serialize = "redact-thinking-2026-02-12")]
+    RedactThinking20260212,
+    #[strum(serialize = "server-side-fallback-2026-06-01")]
+    ServerSideFallback20260601,
+    #[strum(serialize = "server-side-fallback-2026-07-01")]
+    ServerSideFallback20260701,
+    #[strum(serialize = "spend-limit-reads-2026-09-26")]
+    SpendLimitReads20260926,
+    #[strum(serialize = "structured-outputs-2025-12-15")]
+    StructuredOutputs20251215,
+    #[strum(serialize = "task-budgets-2026-03-13")]
+    TaskBudgets20260313,
+    #[strum(serialize = "telemetry-destinations-2026-08-11")]
+    TelemetryDestinations20260811,
+    #[strum(serialize = "thinking-token-count-2026-05-13")]
+    ThinkingTokenCount20260513,
+    #[strum(serialize = "token-counting-2024-11-01")]
+    TokenCounting20241101,
+    #[strum(serialize = "user-profiles-2026-03-24")]
+    UserProfiles20260324,
+    #[strum(serialize = "user-profiles-2026-08-18")]
+    UserProfiles20260818,
+    #[strum(serialize = "user-profiles-2026-09-04")]
+    UserProfiles20260904,
+    #[strum(serialize = "webhooks-2026-10-07")]
+    Webhooks20261007,
     #[strum(default, transparent)]
     Other(String),
 }
 
 impl AnthropicBeta {
-    pub const KNOWN: [Self; 40] = [
+    pub const KNOWN: [Self; 77] = [
         Self::Oauth20250420,
         Self::WebFetch20250910,
         Self::WebSearch20250305,
@@ -170,6 +244,43 @@ impl AnthropicBeta {
         Self::ToolExamples20251029,
         Self::ToolSearchTool20251019,
         Self::InlineTools20260915,
+        Self::AfkMode20260131,
+        Self::AgentMemory20260722,
+        Self::AnalyticsUsageCost20260824,
+        Self::CacheDiagnosis20260407,
+        Self::CePlugins20260901,
+        Self::CeUserManagement20260713,
+        Self::CodeExecution20250522,
+        Self::ComputerUse20241022,
+        Self::DevFullThinking20250514,
+        Self::Dreaming20260421,
+        Self::ExperimentalClaudeTag20261021,
+        Self::ExtendedCacheTtl20250411,
+        Self::FallbackCredit20260601,
+        Self::FallbackCredit20260701,
+        Self::ManagedAgents20260401,
+        Self::McpClient20260915,
+        Self::McpTunnels20260622,
+        Self::MessageBatches20240924,
+        Self::MessageThreads20260812,
+        Self::MidConversationSystemClearAt20260821,
+        Self::ModelContextWindowExceeded20250826,
+        Self::Output300k20260324,
+        Self::Pdfs20240925,
+        Self::PromptCaching20240731,
+        Self::RedactThinking20260212,
+        Self::ServerSideFallback20260601,
+        Self::ServerSideFallback20260701,
+        Self::SpendLimitReads20260926,
+        Self::StructuredOutputs20251215,
+        Self::TaskBudgets20260313,
+        Self::TelemetryDestinations20260811,
+        Self::ThinkingTokenCount20260513,
+        Self::TokenCounting20241101,
+        Self::UserProfiles20260324,
+        Self::UserProfiles20260818,
+        Self::UserProfiles20260904,
+        Self::Webhooks20261007,
     ];
 
     pub fn as_str(&self) -> &str {
@@ -195,10 +306,9 @@ impl AnthropicBeta {
 
     fn rejected_by(&self) -> &'static [BetaProvider] {
         match self {
-            Self::AdvancedToolUse20251120 => {
+            Self::AdvancedToolUse20251120 | Self::ToolSearchTool20251019 => {
                 &[BetaProvider::BedrockConverse, BetaProvider::Databricks]
             }
-            Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
             Self::AdvisorTool20260301 => &[
                 BetaProvider::AzureAi,
                 BetaProvider::BedrockConverse,
@@ -207,98 +317,99 @@ impl AnthropicBeta {
                 BetaProvider::VertexAi,
                 BetaProvider::Databricks,
             ],
-            Self::Compact20260904 => &[
+            Self::AfkMode20260131
+            | Self::CePlugins20260901
+            | Self::CeUserManagement20260713
+            | Self::Compact20260904
+            | Self::Dreaming20260421
+            | Self::ExperimentalClaudeTag20261021
+            | Self::ExtendedCacheTtl20250411
+            | Self::FallbackCredit20260601
+            | Self::FallbackCredit20260701
+            | Self::InlineTools20260915
+            | Self::ManagedAgents20260401
+            | Self::McpTunnels20260622
+            | Self::MidConversationSystemClearAt20260821
+            | Self::ModelContextWindowExceeded20250826
+            | Self::Oauth20250420
+            | Self::Pdfs20240925
+            | Self::PromptCaching20240731
+            | Self::PromptCachingScope20260105
+            | Self::RedactThinking20260212
+            | Self::ServerSideFallback20260601
+            | Self::SpendLimitReads20260926
+            | Self::StructuredOutputs20251215
+            | Self::TaskBudgets20260313
+            | Self::TelemetryDestinations20260811
+            | Self::ThinkingTokenCount20260513
+            | Self::TokenCounting20241101
+            | Self::UserProfiles20260324
+            | Self::UserProfiles20260818
+            | Self::UserProfiles20260904 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::Databricks,
             ],
-            Self::Bash20241022
+            Self::AgentMemory20260722
+            | Self::Bash20241022
             | Self::Bash20250124
+            | Self::DevFullThinking20250514
             | Self::McpServers20251204
             | Self::StructuredOutput20240301
             | Self::TextEditor20241022
             | Self::TextEditor20250124 => BetaProvider::VARIANTS,
-            Self::ClaudeCode20250219 | Self::ToolExamples20251029 => &[
-                BetaProvider::Anthropic,
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::CodeExecution20250825
+            Self::AnalyticsUsageCost20260824
+            | Self::CacheDiagnosis20260407
+            | Self::CodeExecution20250522
+            | Self::CodeExecution20250825
+            | Self::ComputerUse20241022
+            | Self::FastMode20260201
             | Self::FilesApi20250414
             | Self::McpClient20250404
             | Self::McpClient20251120
-            | Self::PromptCachingScope20260105
+            | Self::McpClient20260915
+            | Self::MessageBatches20240924
+            | Self::MessageThreads20260812
+            | Self::Output300k20260324
+            | Self::ServerSideFallback20260701
             | Self::Skills20251002
-            | Self::WebFetch20250910 => &[
+            | Self::WebFetch20250910
+            | Self::Webhooks20261007 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::VertexAi,
                 BetaProvider::Databricks,
             ],
-            Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
+            Self::ClaudeCode20250219
+            | Self::PerTurnControl20260701
+            | Self::ToolExamples20251029 => &[
+                BetaProvider::BedrockConverse,
+                BetaProvider::Bedrock,
+                BetaProvider::Databricks,
+            ],
+            Self::Compact20260112
+            | Self::ContextManagement20250627
+            | Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
             Self::ComputerUse20250124
             | Self::ComputerUse20251124
             | Self::Context1m20250807
+            | Self::Effort20251124
+            | Self::FineGrainedToolStreaming20250514
             | Self::ThinkingBindingControls20260801
             | Self::ThinkingDisplayUpdates20260818 => &[],
-            Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
-            Self::Effort20251124 => &[BetaProvider::VertexAi],
-            Self::FastMode20260201 | Self::Oauth20250420 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::BedrockMantle,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::FineGrainedToolStreaming20250514 => {
-                &[BetaProvider::AzureAi, BetaProvider::VertexAi]
-            }
-            Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
-                &[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
-            }
-            Self::MidConversationOutputConfig20260701 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::BedrockMantle,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::MidConversationToolChanges20260701 => &[
+            Self::InterleavedThinking20250514
+            | Self::Output128k20250219
+            | Self::TokenEfficientTools20250219
+            | Self::WebSearch20250305 => &[BetaProvider::BedrockConverse, BetaProvider::Bedrock],
+            Self::MidConversationOutputConfig20260701
+            | Self::MidConversationToolChanges20260701 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::BedrockMantle,
                 BetaProvider::Databricks,
             ],
-            Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::VertexAi,
-            ],
-            Self::PerTurnControl20260701 => &[
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::Databricks,
-            ],
-            Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
-            Self::ToolSearchTool20251019 => &[
-                BetaProvider::Anthropic,
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Databricks,
-            ],
-            Self::InlineTools20260915 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::BedrockMantle,
-                BetaProvider::Databricks,
-            ],
+            Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock],
             Self::Other(_) => BetaProvider::VARIANTS,
         }
     }
@@ -451,7 +562,44 @@ mod tests {
             AnthropicBeta::TokenEfficientTools20250219,
             AnthropicBeta::ToolExamples20251029,
             AnthropicBeta::ToolSearchTool20251019,
-            AnthropicBeta::InlineTools20260915
+            AnthropicBeta::InlineTools20260915,
+            AnthropicBeta::AfkMode20260131,
+            AnthropicBeta::AgentMemory20260722,
+            AnthropicBeta::AnalyticsUsageCost20260824,
+            AnthropicBeta::CacheDiagnosis20260407,
+            AnthropicBeta::CePlugins20260901,
+            AnthropicBeta::CeUserManagement20260713,
+            AnthropicBeta::CodeExecution20250522,
+            AnthropicBeta::ComputerUse20241022,
+            AnthropicBeta::DevFullThinking20250514,
+            AnthropicBeta::Dreaming20260421,
+            AnthropicBeta::ExperimentalClaudeTag20261021,
+            AnthropicBeta::ExtendedCacheTtl20250411,
+            AnthropicBeta::FallbackCredit20260601,
+            AnthropicBeta::FallbackCredit20260701,
+            AnthropicBeta::ManagedAgents20260401,
+            AnthropicBeta::McpClient20260915,
+            AnthropicBeta::McpTunnels20260622,
+            AnthropicBeta::MessageBatches20240924,
+            AnthropicBeta::MessageThreads20260812,
+            AnthropicBeta::MidConversationSystemClearAt20260821,
+            AnthropicBeta::ModelContextWindowExceeded20250826,
+            AnthropicBeta::Output300k20260324,
+            AnthropicBeta::Pdfs20240925,
+            AnthropicBeta::PromptCaching20240731,
+            AnthropicBeta::RedactThinking20260212,
+            AnthropicBeta::ServerSideFallback20260601,
+            AnthropicBeta::ServerSideFallback20260701,
+            AnthropicBeta::SpendLimitReads20260926,
+            AnthropicBeta::StructuredOutputs20251215,
+            AnthropicBeta::TaskBudgets20260313,
+            AnthropicBeta::TelemetryDestinations20260811,
+            AnthropicBeta::ThinkingTokenCount20260513,
+            AnthropicBeta::TokenCounting20241101,
+            AnthropicBeta::UserProfiles20260324,
+            AnthropicBeta::UserProfiles20260818,
+            AnthropicBeta::UserProfiles20260904,
+            AnthropicBeta::Webhooks20261007
         )]
         beta: AnthropicBeta,
     ) {
@@ -551,7 +699,7 @@ mod tests {
         BetaProvider::Anthropic,
         Some(AnthropicBeta::Oauth20250420)
     )]
-    #[case::rejected("effort-2025-11-24", BetaProvider::VertexAi, None)]
+    #[case::rejected("skills-2025-10-02", BetaProvider::VertexAi, None)]
     #[case::unknown("example-beta-2099-01-01", BetaProvider::Anthropic, None)]
     fn on_resolves_known_spellings_held_as_other(
         #[case] raw: &str,
@@ -580,12 +728,11 @@ mod tests {
 
     #[test]
     fn unknown_values_are_kept_verbatim() {
-        let parsed: AnthropicBeta = "claude-code-20250219".parse().unwrap();
-        assert_eq!(
-            parsed,
-            AnthropicBeta::Other("claude-code-20250219".to_string())
+        let parsed: AnthropicBeta = "example-beta-2099-01-01".parse().unwrap();
+        assert!(
+            matches!(&parsed, AnthropicBeta::Other(value) if value == "example-beta-2099-01-01")
         );
-        assert_eq!(parsed.to_string(), "claude-code-20250219");
+        assert_eq!(parsed.to_string(), "example-beta-2099-01-01");
     }
 
     #[test]

litellm-rust/crates/llms/src/anthropic/beta_headers.rs

@@ -96,7 +96,10 @@ mod tests {
     }
 
     #[rstest]
-    #[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")]
+    #[case::azure_ai(
+        BetaProvider::AzureAi,
+        "advisor-tool-2026-03-01,example-beta-2099-01-01"
+    )]
     #[case::anthropic(BetaProvider::Anthropic, "bash_20241022")]
     #[case::vertex_ai(BetaProvider::VertexAi, " , ")]
     fn filter_removes_header_when_nothing_survives(

litellm-rust/crates/model-catalog/src/model_info.rs

@@ -186,6 +186,9 @@ pub struct ModelInfoFields {
     pub supports_image_size: Option<bool>,
     pub supported_audio_formats: Option<Value>,
     pub token_count_framing: Option<i64>,
+    pub token_count_tool_framing: Option<i64>,
+    pub token_count_responses_framing: Option<i64>,
+    pub token_count_responses_tool_framing: Option<i64>,
     pub token_count_model: Option<Value>,
     pub token_count_tokenizer: Option<Value>,
     pub vertex_ai_audio_api: Option<Value>,

litellm-rust/crates/model-catalog/src/wire/attributes.rs

@@ -11,7 +11,7 @@ use crate::error::{DateCause, Error, Repr};
 use crate::vocabulary::{
     AudioFormat, EffortLevel, Endpoint, InputModality, Mode, OutputModality, VertexAiAudioApi,
 };
-use crate::wire::lax::Count;
+use crate::wire::lax::{Count, NumericCount};
 
 macro_rules! attributes {
     ($($(#[$doc:meta])* $name:ident: $ty:ty $(as $via:ty)?),* $(,)?) => {
@@ -104,9 +104,14 @@ attributes! {
     /// Cloud regions the model is available in ('global' or region ids).
     supported_regions: Vec<String>,
     thinking_always_on: bool,
-    token_count_framing: u64 as Count,
+    /// Whether the model thinks when the request carries no thinking param.
+    thinking_on_by_default: bool,
+    token_count_framing: u64 as NumericCount,
     token_count_model: TokenCountModel,
+    token_count_responses_framing: u64 as NumericCount,
+    token_count_responses_tool_framing: u64 as NumericCount,
     token_count_tokenizer: TokenCountTokenizer,
+    token_count_tool_framing: u64 as NumericCount,
     /// Provider default tokens-per-minute limit.
     tpm: u64 as Count,
     use_openai_responses_path: bool,
@@ -298,10 +303,24 @@ mod tests {
     #[case::known_mode("mode", json!("chat"), true)]
     #[case::unknown_mode("mode", json!("chatt"), false)]
     #[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
+    #[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
+    #[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
     #[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
     #[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
     #[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]
     #[case::reasoning_mode_not_a_string("reasoning_modes", json!([1]), false)]
+    #[case::framing("token_count_framing", json!(12), true)]
+    #[case::framing_written_as_text("token_count_framing", json!("12"), false)]
+    #[case::framing_written_as_whole_float("token_count_framing", json!(12.0), true)]
+    #[case::fractional_tool_framing("token_count_tool_framing", json!(1.5), false)]
+    #[case::tool_framing("token_count_tool_framing", json!(214), true)]
+    #[case::tool_framing_written_as_text("token_count_tool_framing", json!("214"), false)]
+    #[case::tool_framing_as_a_flag("token_count_tool_framing", json!(true), false)]
+    #[case::negative_tool_framing("token_count_tool_framing", json!(-1), false)]
+    #[case::responses_framing("token_count_responses_framing", json!(38), true)]
+    #[case::responses_framing_as_a_flag("token_count_responses_framing", json!(false), false)]
+    #[case::responses_tool_framing("token_count_responses_tool_framing", json!(267), true)]
+    #[case::responses_tool_framing_written_as_text("token_count_responses_tool_framing", json!("267"), false)]
     fn each_attribute_reads_its_own_type(
         #[case] key: &str,
         #[case] value: Value,

litellm-rust/crates/model-catalog/src/wire/lax.rs

@@ -8,6 +8,8 @@ pub(crate) struct LaxI64;
 
 pub(crate) struct Count;
 
+pub(crate) struct NumericCount;
+
 impl<'de> DeserializeAs<'de, i64> for LaxI64 {
     fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {
         deserializer.deserialize_any(Self)
@@ -52,6 +54,34 @@ impl<'de> DeserializeAs<'de, u64> for Count {
     }
 }
 
+impl<'de> DeserializeAs<'de, u64> for NumericCount {
+    fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
+        deserializer.deserialize_any(Self)
+    }
+}
+
+impl Visitor<'_> for NumericCount {
+    type Value = u64;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("a whole number of zero or more")
+    }
+
+    fn visit_u64<E: de::Error>(self, value: u64) -> Result<u64, E> {
+        Ok(value)
+    }
+
+    fn visit_i64<E: de::Error>(self, value: i64) -> Result<u64, E> {
+        u64::try_from(value).map_err(|_| E::invalid_value(de::Unexpected::Signed(value), &self))
+    }
+
+    fn visit_f64<E: de::Error>(self, value: f64) -> Result<u64, E> {
+        integral_float(value)
+            .and_then(|whole| u64::try_from(whole).ok())
+            .ok_or_else(|| E::invalid_value(de::Unexpected::Float(value), &self))
+    }
+}
+
 fn integer_string(value: &str) -> Option<i64> {
     let integer = match value.split_once('.') {
         Some((integer, fraction)) => {
@@ -99,6 +129,12 @@ mod tests {
             .map(DeserializeAsWrap::into_inner)
     }
 
+    fn numeric_count(value: Value) -> Option<u64> {
+        DeserializeAsWrap::<u64, NumericCount>::deserialize(value)
+            .ok()
+            .map(DeserializeAsWrap::into_inner)
+    }
+
     use serde::Deserialize;
 
     #[rstest]
@@ -125,4 +161,23 @@ mod tests {
     ) {
         assert_eq!(count(written), expected);
     }
+
+    #[rstest]
+    #[case::integer(json!(214), Some(214))]
+    #[case::zero(json!(0), Some(0))]
+    #[case::whole_float(json!(214.0), Some(214))]
+    #[case::negative_zero(json!(-0.0), Some(0))]
+    #[case::above_i64(json!(u64::MAX), Some(u64::MAX))]
+    #[case::negative(json!(-1), None)]
+    #[case::negative_whole_float(json!(-1.0), None)]
+    #[case::fraction(json!(1.5), None)]
+    #[case::string(json!("214"), None)]
+    #[case::boolean(json!(true), None)]
+    #[case::list(json!([214]), None)]
+    fn numeric_counts_accept_only_whole_json_numbers(
+        #[case] written: Value,
+        #[case] expected: Option<u64>,
+    ) {
+        assert_eq!(numeric_count(written), expected);
+    }
 }

litellm/__init__.py

@@ -2374,6 +2374,7 @@ if TYPE_CHECKING:
     supports_web_search: Callable[..., bool]
     supports_tool_search: Callable[..., bool]
     supports_url_context: Callable[..., bool]
+    supports_continuation: Callable[..., bool]
     supports_response_schema: Callable[..., bool]
     supports_parallel_function_calling: Callable[..., bool]
     supports_vision: Callable[..., bool]

litellm/_lazy_imports_registry.py

@@ -33,6 +33,7 @@ UTILS_NAMES: Final = (
     "supports_web_search",
     "supports_tool_search",
     "supports_url_context",
+    "supports_continuation",
     "supports_response_schema",
     "supports_parallel_function_calling",
     "supports_vision",
@@ -488,6 +489,7 @@ _UTILS_IMPORT_MAP: Final = {
     "supports_web_search": (".utils", "supports_web_search"),
     "supports_tool_search": (".utils", "supports_tool_search"),
     "supports_url_context": (".utils", "supports_url_context"),
+    "supports_continuation": (".utils", "supports_continuation"),
     "supports_response_schema": (".utils", "supports_response_schema"),
     "supports_parallel_function_calling": (
         ".utils",

litellm/anthropic_beta_headers_config.json

@@ -37,7 +37,47 @@
     "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
     "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
+    "cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": "code-execution-2025-05-22",
+    "computer-use-2024-10-22": "computer-use-2024-10-22",
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-client-2026-09-15": "mcp-client-2026-09-15",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "message-batches-2024-09-24": "message-batches-2024-09-24",
+    "message-threads-2026-08-12": "message-threads-2026-08-12",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "output-300k-2026-03-24": "output-300k-2026-03-24",
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04",
+    "webhooks-2026-10-07": "webhooks-2026-10-07",
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "azure_ai": {
     "advisor-tool-2026-03-01": null,
@@ -45,23 +85,21 @@
     "bash_20241022": null,
     "bash_20250124": null,
     "code-execution-2025-08-25": "code-execution-2025-08-25",
-    "compact-2026-01-12": null,
-    "compact-2026-09-04": "compact-2026-09-04",
+    "compact-2026-01-12": "compact-2026-01-12",
     "computer-use-2025-01-24": "computer-use-2025-01-24",
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
     "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
-    "fast-mode-2026-02-01": null,
+    "fast-mode-2026-02-01": "fast-mode-2026-02-01",
     "files-api-2025-04-14": "files-api-2025-04-14",
-    "fine-grained-tool-streaming-2025-05-14": null,
+    "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
     "mcp-client-2025-11-20": "mcp-client-2025-11-20",
     "mcp-client-2025-04-04": "mcp-client-2025-04-04",
     "mcp-servers-2025-12-04": null,
-    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "output-128k-2025-02-19": null,
+    "output-128k-2025-02-19": "output-128k-2025-02-19",
     "structured-output-2024-03-01": null,
     "per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
     "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
@@ -71,9 +109,54 @@
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
-    "token-efficient-tools-2025-02-19": null,
+    "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "web-fetch-2025-09-10": "web-fetch-2025-09-10",
-    "web-search-2025-03-05": "web-search-2025-03-05"
+    "web-search-2025-03-05": "web-search-2025-03-05",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
+    "cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": "code-execution-2025-05-22",
+    "compact-2026-09-04": "compact-2026-09-04",
+    "computer-use-2024-10-22": "computer-use-2024-10-22",
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-client-2026-09-15": "mcp-client-2026-09-15",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "message-batches-2024-09-24": "message-batches-2024-09-24",
+    "message-threads-2026-08-12": "message-threads-2026-08-12",
+    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "oauth-2025-04-20": "oauth-2025-04-20",
+    "output-300k-2026-03-24": "output-300k-2026-03-24",
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04",
+    "webhooks-2026-10-07": "webhooks-2026-10-07",
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "bedrock_converse": {
     "advisor-tool-2026-03-01": null,
@@ -196,34 +279,92 @@
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
     "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
-    "effort-2025-11-24": null,
+    "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
-    "fine-grained-tool-streaming-2025-05-14": null,
+    "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
     "mcp-client-2025-11-20": null,
     "mcp-client-2025-04-04": null,
     "mcp-servers-2025-12-04": null,
-    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "output-128k-2025-02-19": null,
+    "output-128k-2025-02-19": "output-128k-2025-02-19",
     "structured-output-2024-03-01": null,
     "per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
     "prompt-caching-scope-2026-01-05": null,
     "skills-2025-10-02": null,
-    "structured-outputs-2025-11-13": null,
+    "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
     "text_editor_20241022": null,
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
-    "token-efficient-tools-2025-02-19": null,
+    "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
     "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": "web-search-2025-03-05",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": null,
+    "cache-diagnosis-2026-04-07": null,
+    "ce-plugins-2026-09-01": null,
+    "ce-user-management-2026-07-13": null,
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": null,
+    "compact-2026-09-04": null,
+    "computer-use-2024-10-22": null,
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": null,
+    "experimental-claude-tag-2026-10-21": null,
+    "extended-cache-ttl-2025-04-11": null,
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "managed-agents-2026-04-01": null,
+    "mcp-client-2026-09-15": null,
+    "mcp-tunnels-2026-06-22": null,
+    "message-batches-2024-09-24": null,
+    "message-threads-2026-08-12": null,
+    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "oauth-2025-04-20": null,
+    "output-300k-2026-03-24": null,
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": null,
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": null,
+    "spend-limit-reads-2026-09-26": null,
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": null,
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "user-profiles-2026-03-24": null,
+    "user-profiles-2026-08-18": null,
+    "user-profiles-2026-09-04": null,
+    "webhooks-2026-10-07": null,
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": null,
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "vertex_ai_header_only": {
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
     "compact-2026-09-04": "compact-2026-09-04",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "oauth-2025-04-20": "oauth-2025-04-20",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04"
   },
   "databricks": {
     "advisor-tool-2026-03-01": null,

litellm/anthropic_beta_headers_manager.py

@@ -31,6 +31,7 @@ from concurrent.futures import ThreadPoolExecutor
 from functools import cache
 from importlib.resources import files
 from typing import Final, Literal, TypeAlias
+from urllib.parse import urlparse
 
 import httpx
 from pydantic import TypeAdapter
@@ -48,6 +49,7 @@ _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
 _DECODED_JSON: Final = TypeAdapter(object)
 
 BetaPlacement: TypeAlias = Literal["header", "body"]
+UnlistedBetas: TypeAlias = Literal["drop", "forward"]
 _HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only"
 
 
@@ -280,12 +282,13 @@ def filter_and_transform_beta_headers(
     beta_headers: list[str],
     provider: str,
     placement: BetaPlacement = "header",
+    unlisted: UnlistedBetas = "drop",
 ) -> list[str]:
     """
     Filter and transform beta headers based on provider's mapping configuration.
 
     This function:
-    1. Only allows headers that are present in the provider's mapping keys
+    1. Only allows headers that are present in the provider's mapping keys, unless ``unlisted`` is "forward"
     2. Filters out headers with null values (unsupported)
     3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool)
 
@@ -293,6 +296,7 @@ def filter_and_transform_beta_headers(
         beta_headers: List of Anthropic beta header values
         provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
         placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field
+        unlisted: Whether a beta missing from the mapping is dropped or forwarded as sent
 
     Returns:
         List of filtered and transformed beta headers for the provider
@@ -307,6 +311,10 @@ def filter_and_transform_beta_headers(
     for header in beta_headers:
         header = header.strip()
 
+        if header not in provider_mapping and unlisted == "forward":
+            filtered_headers.add(header)
+            continue
+
         # Check if header is in the mapping
         if header not in provider_mapping:
             verbose_logger.debug(
@@ -388,9 +396,38 @@ def get_provider_beta_header(
     return provider_mapping[anthropic_beta_header]
 
 
+FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = (
+    "services.ai.azure.com",
+    "openai.azure.com",
+    "cognitiveservices.azure.com",
+)
+
+
+def is_foundry_api_base(api_base: str | None) -> bool:
+    """Whether ``api_base`` points at an Azure AI Foundry host, where Claude is served by Microsoft."""
+    if not api_base:
+        return False
+    host: Final = urlparse(api_base if "://" in api_base else f"https://{api_base}").hostname or ""
+    return any(host == suffix or host.endswith(f".{suffix}") for suffix in FOUNDRY_API_BASE_HOST_SUFFIXES)
+
+
+def resolve_beta_provider(provider: str, api_base: str | None = None) -> str:
+    """
+    Pick the beta allowlist row for one deployment, not one provider string.
+
+    An ``anthropic`` deployment whose ``api_base`` is a Foundry host is served by Microsoft and
+    gets the ``azure_ai`` row, since Foundry accepts a different set of betas than api.anthropic.com.
+    """
+    if provider == "anthropic" and is_foundry_api_base(api_base):
+        return "azure_ai"
+    return provider
+
+
 def update_headers_with_filtered_beta(
     headers: dict,
     provider: str,
+    api_base: str | None = None,
+    unlisted: UnlistedBetas = "drop",
 ) -> dict:
     """
     Update headers dict by filtering and transforming anthropic-beta header values.
@@ -399,6 +436,8 @@ def update_headers_with_filtered_beta(
     Args:
         headers: Request headers dict (will be modified in place)
         provider: Provider name
+        api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
+        unlisted: Whether a beta missing from the row is dropped or forwarded as sent
 
     Returns:
         Updated headers dict
@@ -413,7 +452,8 @@ def update_headers_with_filtered_beta(
     # Filter and transform based on provider
     filtered_beta_values: Final = filter_and_transform_beta_headers(
         beta_headers=beta_values,
-        provider=provider,
+        provider=resolve_beta_provider(provider, api_base),
+        unlisted=unlisted,
     )
 
     # Update or remove the header
@@ -430,6 +470,7 @@ def update_request_with_filtered_beta(
     headers: dict,
     request_data: dict,
     provider: str,
+    api_base: str | None = None,
 ) -> tuple[dict[str, str], dict[str, object]]:
     """
     Update both headers and request body beta fields based on provider support.
@@ -439,11 +480,12 @@ def update_request_with_filtered_beta(
         headers: Request headers dict (will be modified in place)
         request_data: Request body dict (will be modified in place)
         provider: Provider name
+        api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``)
 
     Returns:
         Tuple of (updated headers, updated request_data)
     """
-    headers = update_headers_with_filtered_beta(headers=headers, provider=provider)
+    headers = update_headers_with_filtered_beta(headers=headers, provider=provider, api_base=api_base)
 
     existing_body_betas: Final = request_data.get("anthropic_beta")
     if not existing_body_betas:
@@ -451,7 +493,7 @@ def update_request_with_filtered_beta(
 
     filtered_body_betas: Final = filter_and_transform_beta_headers(
         beta_headers=existing_body_betas,
-        provider=provider,
+        provider=resolve_beta_provider(provider, api_base),
         placement="body",
     )
 

litellm/anthropic_interface/exceptions/exception_mapping_utils.py

@@ -16,7 +16,7 @@ from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicE
 _PARSED_ERROR_BODY: Final = TypeAdapter(object)
 
 # HTTP status code -> Anthropic error type
-# Source: https://docs.anthropic.com/en/api/errors
+# Source: https://platform.claude.com/docs/en/api/errors
 ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
     400: "invalid_request_error",
     401: "authentication_error",
@@ -46,6 +46,7 @@ class AnthropicExceptionMapping:
         status_code: int,
         message: str,
         request_id: str | None = None,
+        error_type: AnthropicErrorType | None = None,
     ) -> AnthropicErrorResponse:
         """
         Create an Anthropic-formatted error response dict.
@@ -57,12 +58,10 @@ class AnthropicExceptionMapping:
             "request_id": "req_..."
         }
         """
-        error_type: Final = AnthropicExceptionMapping.get_error_type(status_code)
-
         response: Final[AnthropicErrorResponse] = {
             "type": "error",
             "error": {
-                "type": error_type,
+                "type": error_type or AnthropicExceptionMapping.get_error_type(status_code),
                 "message": message,
             },
         }
@@ -131,6 +130,7 @@ class AnthropicExceptionMapping:
         status_code: int,
         raw_message: str,
         request_id: str | None = None,
+        error_type: AnthropicErrorType | None = None,
     ) -> AnthropicErrorResponse:
         """
         Transform an error message to Anthropic format.
@@ -144,6 +144,7 @@ class AnthropicExceptionMapping:
             status_code: HTTP status code
             raw_message: Raw error message (may be JSON string or plain text)
             request_id: Optional request ID to include
+            error_type: Anthropic error type to use instead of the one the status maps to
 
         Returns:
             AnthropicErrorResponse dict
@@ -169,6 +170,7 @@ class AnthropicExceptionMapping:
             status_code=status_code,
             message=message,
             request_id=request_id,
+            error_type=error_type,
         )
 
 

litellm/anthropic_interface/exceptions/exceptions.py

@@ -6,15 +6,18 @@ from typing import Literal
 from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
 
 # Known Anthropic error types
-# Source: https://docs.anthropic.com/en/api/errors
+# Source: https://platform.claude.com/docs/en/api/errors
 AnthropicErrorType = Literal[
     "invalid_request_error",
     "authentication_error",
+    "billing_error",
     "permission_error",
     "not_found_error",
+    "conflict_error",
     "request_too_large",
     "rate_limit_error",
     "api_error",
+    "timeout_error",
     "overloaded_error",
 ]
 
@@ -25,6 +28,7 @@ class AnthropicErrorDetail(TypedDict):
     type: AnthropicErrorType
     message: str
     provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]]
+    litellm_error_type: NotRequired[ReadOnly[str]]
     litellm_call_id: NotRequired[ReadOnly[str]]
 
 

litellm/caching/caching.py

@@ -45,13 +45,27 @@ from .redis_semantic_cache import RedisSemanticCache
 from .s3_cache import S3Cache
 
 RESPONSE_CACHE_TARGET: Final = "llm_response"
-_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
+_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token", "return_token_ids"})
 _DECODED_LITERAL: Final = TypeAdapter(object)
 _REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
     list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
 )
 
 
+def _keyed_provider_params_in_extra_body(kwargs: Mapping[str, object]) -> str:
+    raw_extra_body: Final = kwargs.get("extra_body")
+    if not isinstance(raw_extra_body, dict):
+        return ""
+    extra_body: Final = cast(  # cast-ok: isinstance leaves the key and value types unknown
+        Mapping[str, object], raw_extra_body
+    )
+    return "".join(
+        f"extra_body.{param}: {extra_body[param]}"
+        for param in sorted(_ALWAYS_KEYED_PROVIDER_PARAMS)
+        if extra_body.get(param)
+    )
+
+
 @contextmanager
 def response_cache_phase(operation: Literal["get", "set"]) -> Generator[None]:
     """The ``cache.get llm_response`` / ``cache.set llm_response`` span a response-cache read or write runs
@@ -429,7 +443,10 @@ class Cache:
             "" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
         )
         hashed_cache_key = Cache._get_hashed_cache_key(
-            cache_key + reasoning_summary_material + tool_call_id_shape_material
+            cache_key
+            + _keyed_provider_params_in_extra_body(kwargs)
+            + reasoning_summary_material
+            + tool_call_id_shape_material
         )
         hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
         verbose_logger.debug(

litellm/caching/dual_cache.py

@@ -673,7 +673,7 @@ class DualCache(BaseCache):
         if self.redis_cache is not None:
             self.redis_cache.flush_cache()
 
-    def delete_cache(self, key):
+    def delete_cache(self, key: str) -> None:
         """
         Delete a key from the cache
         """

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -34,10 +34,12 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
     requested_phase,
 )
 from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
-    prompt_cache_breakpoint_for_wire,
+    prompt_cache_breakpoint_with_readable_mode,
     responses_reasoning_items_from_thinking_blocks,
+    whole_prompt_cache_breakpoint_or_none,
     with_prompt_cache_breakpoint,
 )
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
@@ -97,6 +99,12 @@ _RESPONSES_API_ONLY_FIELDS: Final = frozenset((*Response.model_fields, *Response
 )
 
 
+def _bridged_prompt_cache_breakpoint(marker: object, drop_params: bool, target_trims_breakpoints: bool) -> object:
+    if target_trims_breakpoints:
+        return prompt_cache_breakpoint_with_readable_mode(marker, drop_params)
+    return whole_prompt_cache_breakpoint_or_none(marker) if drop_params else marker
+
+
 def _strip_prompt_cache_breakpoint_from_content_block(value: object) -> object:
     if not isinstance(value, dict):
         return value
@@ -844,6 +852,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         messages: list["AllMessageValues"],
         *,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
         keep_prompt_cache_breakpoints: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
@@ -852,6 +861,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
             messages,
             drop_params=drop_params,
+            target_trims_breakpoints=target_trims_breakpoints,
             replay_tool_search=replay_tool_search,
             include_namespace=include_namespace,
             replay_openai_history=replay_openai_history,
@@ -868,6 +878,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         messages: list["AllMessageValues"],
         *,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
         replay_openai_history: bool = False,
@@ -921,6 +932,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                 content,
                                 role,
                                 drop_params=drop_params,
+                                target_trims_breakpoints=target_trims_breakpoints,
                             ),
                         }
                     )
@@ -940,6 +952,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         content,
                         "user",  # Use "user" role to get input_* types
                         drop_params=drop_params,
+                        target_trims_breakpoints=target_trims_breakpoints,
                     )
                 else:
                     # Fallback: convert unexpected types to input_text
@@ -968,7 +981,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             "type": "message",
                             "role": "assistant",
                             "content": self._convert_content_to_responses_format(
-                                content, "assistant", drop_params=drop_params
+                                content,
+                                "assistant",
+                                drop_params=drop_params,
+                                target_trims_breakpoints=target_trims_breakpoints,
                             ),
                             **_assistant_phase_field(msg, replay_openai_history, drop_params),
                         }
@@ -1009,7 +1025,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         "type": "message",
                         "role": role,
                         "content": self._convert_content_to_responses_format(
-                            content, cast(str, role), drop_params=drop_params
+                            content,
+                            cast(str, role),
+                            drop_params=drop_params,
+                            target_trims_breakpoints=target_trims_breakpoints,
                         ),
                         **_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
                     }
@@ -1135,6 +1154,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
             messages,
             drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
+            target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")),
             keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
             replay_tool_search=_requests_tool_search(cast(object, optional_params)),  # cast-ok: upcast
             include_namespace=True,
@@ -1640,6 +1660,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         | None,
         role: str,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
     ) -> list[dict[str, object]]:
         """Convert chat completion content to responses API format"""
         from litellm.types.llms.openai import ChatCompletionImageObject
@@ -1666,7 +1687,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     if original_type == "text":
                         converted = with_prompt_cache_breakpoint(
                             self._convert_content_str_to_input_text(item.get("text", ""), role),
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   text -> %s", converted)
@@ -1679,14 +1702,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                     cast(ChatCompletionImageObject, item), role
                                 ),
                             ),
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   image_url -> %s", converted)
                     elif original_type == "input_audio":
                         converted = with_prompt_cache_breakpoint(
                             {"type": "input_audio", "input_audio": item.get("input_audio")},
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   input_audio -> %s", converted)
@@ -1702,7 +1729,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                 _input_file_from_file_value(
                                     cast("ChatCompletionFileObject", item).get("file"),  # cast-ok: type tag checked
                                 ),
-                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                _bridged_prompt_cache_breakpoint(
+                                    item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                                ),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   file -> %s", converted)
@@ -1726,7 +1755,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             # Default to input_text for unknown types
                             converted = with_prompt_cache_breakpoint(
                                 self._convert_content_str_to_input_text(str(item.get("text", item)), role),
-                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                _bridged_prompt_cache_breakpoint(
+                                    item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                                ),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   unknown(%s) -> %s", original_type, converted)

litellm/constants.py

@@ -699,6 +699,9 @@ STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
 STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
 STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
 STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
+STREAM_FINISH_CHOICE_FIELDS: Final = ("stop_reason",)
+STREAM_CHUNK_CHOICE_FIELDS: Final = ("token_ids",)
+VLLM_CHOICE_FIELDS: Final = STREAM_FINISH_CHOICE_FIELDS + STREAM_CHUNK_CHOICE_FIELDS
 ### SPEND TRACKING ###
 DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float(
     "DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400

litellm/cost_calculator.py

@@ -6,6 +6,7 @@ import time
 from collections.abc import Mapping, Sequence
 from datetime import datetime
 from functools import lru_cache
+from itertools import accumulate
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
 
@@ -3850,44 +3851,65 @@ def handle_realtime_transcription_cost_calculation(
       - {"type": "duration", "seconds": <float>}  → priced via input_cost_per_second
       - {"type": "tokens", "input_tokens": ...}    → priced via input/audio token cost
     """
-    completed_events: Final = [
-        cast(dict, result) for result in results if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
-    ]
-    if not completed_events:
+    configured_models: Final = tuple(accumulate(map(_session_transcription_model, results), _newer_model))
+    clips: Final = tuple(
+        (as_str_mapping(result.get("usage")) or {}, configured)
+        for result, configured in zip(results, configured_models, strict=True)
+        if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
+    )
+    if not clips:
         return 0.0
 
-    model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
-    model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
+    fallback_model: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
+    clip_models: Final = tuple((usage, configured or fallback_model) for usage, configured in clips)
+    model_infos: Final = MappingProxyType(
+        {model: _cost_map_model_info(model, custom_llm_provider) for model in frozenset(m for _, m in clip_models)}
+    )
     override_info: Final = (
         _cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
     )
+    return sum(
+        (_transcription_usage_cost(dict(usage), model_infos[model], override_info) for usage, model in clip_models),
+        0.0,
+    )
 
-    total_cost = 0.0
-    for event in completed_events:
-        usage = event.get("usage") or {}
-        total_cost += _transcription_usage_cost(usage, model_info, override_info)
-    return total_cost
+
+_TRANSCRIPTION_SESSION_EVENT_TYPES: Final = frozenset(
+    {"transcription_session.created", "transcription_session.updated", "session.created", "session.updated"}
+)
+
+
+def _str_field(mapping: Mapping[str, object] | None, key: str) -> str | None:
+    value: Final = None if mapping is None else mapping.get(key)
+    return value if isinstance(value, str) and value else None
+
+
+def _configured_transcription_model(session: Mapping[str, object]) -> str | None:
+    audio_input: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("input"))
+    nested: Final = _str_field(as_str_mapping((audio_input or {}).get("transcription")), "model")
+    return nested or _str_field(as_str_mapping(session.get("input_audio_transcription")), "model")
+
+
+def _session_of(result: Mapping[str, object]) -> Mapping[str, object] | None:
+    return as_str_mapping(result.get("session")) if result.get("type") in _TRANSCRIPTION_SESSION_EVENT_TYPES else None
+
+
+def _session_transcription_model(result: Mapping[str, object]) -> str | None:
+    session: Final = _session_of(result)
+    return None if session is None else _configured_transcription_model(session)
+
+
+def _newer_model(current: str | None, newer: str | None) -> str | None:
+    return newer or current
 
 
 def _get_transcription_model_name_from_results(
     results: OpenAIRealtimeStreamList,
 ) -> str | None:
-    """Resolve the ASR model from a transcription_session.* / session.* event."""
-    for result in results:
-        if result.get("type") in (
-            "transcription_session.created",
-            "transcription_session.updated",
-            "session.created",
-            "session.updated",
-        ):
-            session = cast(dict, result).get("session", {}) or {}
-            transcription = ((session.get("audio", {}) or {}).get("input", {}) or {}).get(
-                "transcription", {}
-            ) or session.get("input_audio_transcription", {})
-            model = (transcription or {}).get("model") or session.get("model")
-            if model:
-                return model
-    return None
+    return next(
+        (model for result in results if (model := _str_field(_session_of(result), "model"))),
+        None,
+    )
 
 
 _TranscriptionRateKey: TypeAlias = Literal[

litellm/integrations/anthropic_cache_control_hook.py

@@ -17,7 +17,7 @@ from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Se
 from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
 from urllib.parse import urlparse
 
-from pydantic import TypeAdapter, ValidationError
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
@@ -73,7 +73,7 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
 
 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
 OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
-_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
+_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+)|-(0|[1-9]\d?)(?![\da-z]))?")
 _FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
 _O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
 OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
@@ -335,6 +335,30 @@ def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str
     return _named_model_breakpoint_support(model)
 
 
+class _DeclaredBaseModels(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
+
+    base_model: object = None
+    model_info: object = None
+
+    def names(self) -> tuple[str, ...]:
+        model_info: Final = _validated_object_mapping(self.model_info) or {}
+        candidates: Final = (self.base_model, model_info.get("base_model"))
+        return tuple(name for name in candidates if isinstance(name, str) and name)
+
+
+def deployment_prompt_cache_breakpoint_support(
+    model: str, custom_llm_provider: str, litellm_params: object
+) -> bool | None:
+    """``openai_prompt_cache_breakpoint_support`` for a deployment, read from its declared base model first."""
+    declared: Final = _DeclaredBaseModels.model_validate(litellm_params)
+    supports: Final = (
+        openai_prompt_cache_breakpoint_support(name, custom_llm_provider)
+        for name in dict.fromkeys((*declared.names(), model))
+    )
+    return next((support for support in supports if support is not None), None)
+
+
 def _named_model_breakpoint_support(model: str) -> bool | None:
     bare_model: Final = model.rsplit("/", 1)[-1].lower()
     if _O_SERIES_PATTERN.match(bare_model) is not None:
@@ -342,7 +366,7 @@ def _named_model_breakpoint_support(model: str) -> bool | None:
     version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
     if version_match is None:
         return None
-    version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
+    version: Final = (int(version_match.group(1)), int(version_match.group(2) or version_match.group(3) or 0))
     return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
 
 

litellm/integrations/azure_storage/azure_storage.py

@@ -1,7 +1,8 @@
 import asyncio
 import os
 import time
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from functools import cache
 from typing import Final
@@ -15,6 +16,7 @@ from litellm.constants import (
     AZURE_STORAGE_MSFT_VERSION,
 )
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.azure.common_utils import get_azure_ad_token_from_entra_id
 from litellm.llms.custom_httpx.http_handler import (
@@ -47,6 +49,20 @@ def adls_safe_file_name(payload_id: str | None) -> str:
     return f"{name.translate(_ADLS_SAFE_NAME)}.json"
 
 
+@dataclass(frozen=True, slots=True)
+class AzureStorageLogQueueItem:
+    payload: StandardLoggingPayload
+    file_name: str
+
+
+def _queued_log(kwargs: Mapping[str, object], payload: StandardLoggingPayload) -> AzureStorageLogQueueItem:
+    payload_id: Final = payload.get("id")
+    return AzureStorageLogQueueItem(
+        payload=payload,
+        file_name=adls_safe_file_name(payload_id and logged_event_key(kwargs, payload_id)),
+    )
+
+
 @cache
 def _cached_credential_chain_token_provider() -> Callable[[], str]:
     return get_azure_ad_token_provider(
@@ -99,7 +115,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
 
             asyncio.create_task(self.periodic_flush())
             self.flush_lock = asyncio.Lock()
-            self.log_queue: list[StandardLoggingPayload] = []
+            self.log_queue: list[AzureStorageLogQueueItem] = []
             super().__init__(**kwargs, flush_lock=self.flush_lock)
         except Exception as e:
             verbose_logger.exception(
@@ -115,7 +131,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
     def azure_storage_blob_endpoint(self) -> str:
         return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}"
 
-    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+    async def async_log_success_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
         """
         Async Log success events to Azure Blob Storage
 
@@ -133,12 +149,12 @@ class AzureBlobStorageLogger(CustomBatchLogger):
             if standard_logging_payload is None:
                 raise ValueError("standard_logging_payload is not set")
 
-            self.log_queue.append(standard_logging_payload)
+            self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
 
         except Exception as e:
             verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
 
-    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+    async def async_log_failure_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime):
         """
         Async Log failure events to Azure Blob Storage
 
@@ -156,7 +172,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
             if standard_logging_payload is None:
                 raise ValueError("standard_logging_payload is not set")
 
-            self.log_queue.append(standard_logging_payload)
+            self.log_queue.append(_queued_log(kwargs, standard_logging_payload))
         except Exception as e:
             verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e)
 
@@ -195,13 +211,15 @@ class AzureBlobStorageLogger(CustomBatchLogger):
                 len(self.log_queue),
             )
 
-            for payload in self.log_queue:
-                await self.async_upload_payload_to_azure_blob_storage(payload=payload)
+            for item in self.log_queue:
+                await self.async_upload_payload_to_azure_blob_storage(payload=item.payload, file_name=item.file_name)
 
         except Exception as e:
             verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e)
 
-    async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload):
+    async def async_upload_payload_to_azure_blob_storage(
+        self, payload: StandardLoggingPayload, file_name: str | None = None
+    ):
         """
         Uploads the payload to Azure Blob Storage using a 3-step process:
         1. Create file resource
@@ -210,14 +228,14 @@ class AzureBlobStorageLogger(CustomBatchLogger):
         """
         try:
             if self.azure_storage_account_key:
-                await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload)
+                await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload, file_name=file_name)
             else:
                 # Get a valid token instead of always requesting a new one
                 await self.set_valid_azure_ad_token()
                 async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
                 json_payload: Final = safe_dumps(payload) + "\n"  # Add newline for each log entry
                 payload_bytes: Final = json_payload.encode("utf-8")
-                filename: Final = adls_safe_file_name(payload.get("id"))
+                filename: Final = file_name or adls_safe_file_name(payload.get("id"))
                 base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}"
 
                 # Execute the 3-step upload process
@@ -377,7 +395,9 @@ class AzureBlobStorageLogger(CustomBatchLogger):
             self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS
         return self._service_client
 
-    async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload):
+    async def upload_to_azure_data_lake_with_azure_account_key(
+        self, payload: StandardLoggingPayload, file_name: str | None = None
+    ):
         """
         Uploads the payload to Azure Data Lake using the Azure SDK
 
@@ -386,11 +406,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
 
         try:
             today: Final = datetime.now().strftime("%Y-%m-%d")
-            file_name: Final = adls_safe_file_name(payload.get("id"))
+            resolved_file_name: Final = file_name or adls_safe_file_name(payload.get("id"))
             await self._upload_bytes_with_account_key(
-                file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8")
+                file_path=f"{today}/{resolved_file_name}", content=safe_dumps(payload).encode("utf-8")
             )
-            verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name)
+            verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, resolved_file_name)
 
         except Exception as e:
             verbose_logger.exception("Error occurred: %s", e)

litellm/integrations/clickhouse/clickhouse_spend_logger.py

@@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
 from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
 from litellm.integrations.clickhouse.context import is_lens_analysis
 from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
 from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
 from litellm.tracing.types import SpendLogPayload, SpendLogRecord
@@ -134,7 +135,7 @@ def spend_log_row_from_payload(
     usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({})
     cache_read_tokens, cache_write_tokens = _cache_tokens(usage)
     trace_id, span_id = _find_traceparent(metadata, kwargs)
-    request_id = str(payload.get("id") or "")
+    response_id: Final = str(payload.get("id") or "")
     redact = litellm.turn_off_message_logging is True
     track_end_user: Final = not litellm.disable_end_user_cost_tracking
     completion_start_ms = _to_ms(payload.get("completionStartTime"))
@@ -157,8 +158,8 @@ def spend_log_row_from_payload(
         )
     )
     return SpendLogRecord(
-        request_id=request_id,
-        response_id=strip_cache_hit_suffix(request_id),
+        request_id=logged_event_key(kwargs, response_id),
+        response_id=strip_cache_hit_suffix(response_id),
         provider_request_id=(
             get_provider_request_id(kwargs.get("response_headers"))
             or get_provider_request_id(hidden_params.get("additional_headers"))

litellm/integrations/deepeval/deepeval.py

@@ -15,6 +15,7 @@ from litellm.integrations.deepeval.utils import (
     to_zod_compatible_iso,
     validate_environment,
 )
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 
 
 # This file includes the custom callbacks for LiteLLM Proxy
@@ -109,8 +110,10 @@ class DeepEvalLogger(CustomLogger):
             )
         else:
             output = str(standard_logging_object.get("error_string", ""))
+        logged_id: Final = standard_logging_object.get("id", uuid.uuid4())
+        span_id: Final = logged_event_key(kwargs, logged_id) if isinstance(logged_id, str) else logged_id
         return BaseApiSpan(
-            uuid=standard_logging_object.get("id", uuid.uuid4()),
+            uuid=span_id,
             name=("litellm_success_callback" if is_success else "litellm_failure_callback"),
             status=(TraceSpanApiStatus.SUCCESS if is_success else TraceSpanApiStatus.ERRORED),
             type=SpanApiType.LLM,

litellm/integrations/dynamodb.py

@@ -9,6 +9,8 @@ from typing import Final, Protocol
 
 import litellm
 from litellm._uuid import uuid
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 
 
 class _DynamoTable(Protocol):
@@ -22,11 +24,13 @@ class _DynamoResource(Protocol):
 class DyanmoDBLogger:
     # Class variables or attributes
 
-    def __init__(self):
+    def __init__(self, dynamodb: _DynamoResource | None = None):
         # Instance variables
         import boto3
 
-        self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"])
+        self.dynamodb: Final[_DynamoResource] = dynamodb or boto3.resource(
+            "dynamodb", region_name=os.environ["AWS_REGION_NAME"]
+        )
         if litellm.dynamodb_table_name is None:
             raise ValueError(
                 "LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=<your-table>`"
@@ -45,23 +49,23 @@ class DyanmoDBLogger:
 
     _async_log_event = async_log_event
 
-    def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
+    def log_event(self, kwargs: Mapping[str, object], response_obj, start_time, end_time, print_verbose):
         try:
             print_verbose(f"DynamoDB Logging - Enters logging function for model {kwargs}")
 
             # construct payload to send to DynamoDB
             # follows the same params as langfuse.py
-            litellm_params: Final = kwargs.get("litellm_params", {})
-            metadata: Final = litellm_params.get("metadata", {}) or {}  # if litellm_params['metadata'] == None
+            litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
+            metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
             messages: Final = kwargs.get("messages")
             optional_params: Final = kwargs.get("optional_params", {})
             call_type: Final = kwargs.get("call_type", "litellm.completion")
             usage: Final = response_obj["usage"]
-            id: Final = response_obj.get("id", str(uuid.uuid4()))
+            response_id: Final = response_obj.get("id", str(uuid.uuid4()))
 
             # Build the initial payload
             payload: Final[dict[str, object]] = {
-                "id": id,
+                "id": logged_event_key(kwargs, response_id) if isinstance(response_id, str) else response_id,
                 "call_type": call_type,
                 "startTime": start_time,
                 "endTime": end_time,

litellm/integrations/gcs_bucket/gcs_bucket.py

@@ -17,6 +17,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
     sanitize_cloud_object_component,
 )
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
 from litellm.types.integrations.gcs_bucket import *
 from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@@ -296,30 +297,28 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
         else:
             await self._send_individual_logs(items_to_process)
 
-    def _get_object_name(self, kwargs: dict, logging_payload: StandardLoggingPayload, response_obj: Any) -> str:
+    def _get_object_name(
+        self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
+    ) -> str:
         """
         Get the object name to use for the current payload
         """
         current_date: Final = self._get_object_date_from_datetime(datetime.now(timezone.utc))
+        metadata: Final = as_str_mapping((as_str_mapping(kwargs.get("litellm_params")) or {}).get("metadata")) or {}
+        gcs_log_id: Final = metadata.get("gcs_log_id")
+        safe_log_id: Final = sanitize_cloud_object_component(
+            gcs_log_id if isinstance(gcs_log_id, str) else None, fallback=""
+        )
+        if safe_log_id:
+            return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
         if logging_payload.get("error_str", None) is not None:
-            object_name = self._generate_failure_object_name(
-                request_date_str=current_date,
-            )
-        else:
-            object_name = self._generate_success_object_name(
-                request_date_str=current_date,
-                response_id=response_obj.get("id", ""),
-            )
-
-        # used for testing
-        _litellm_params: Final = kwargs.get("litellm_params", None) or {}
-        _metadata: Final = _litellm_params.get("metadata", None) or {}
-        if "gcs_log_id" in _metadata:
-            safe_log_id: Final = sanitize_cloud_object_component(_metadata.get("gcs_log_id"), fallback="")
-            if safe_log_id:
-                object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
-
-        return object_name
+            return self._generate_failure_object_name(request_date_str=current_date)
+        raw_response_id: Final[object] = response_obj.get("id", "")
+        response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
+        return self._generate_success_object_name(
+            request_date_str=current_date,
+            response_id=logged_event_key(kwargs, response_id),
+        )
 
     async def get_request_response_payload(
         self,

litellm/integrations/openmeter.py

@@ -3,12 +3,15 @@
 
 import json
 import os
+from collections.abc import Mapping
 from typing import Final
 
 import httpx
 
 import litellm
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.llms.custom_httpx.http_handler import (
     HTTPHandler,
     get_async_httpx_client,
@@ -48,7 +51,7 @@ class OpenMeterLogger(CustomLogger):
         if len(missing_keys) > 0:
             raise Exception(f"Missing keys={missing_keys} in environment.")
 
-    def _common_logic(self, kwargs: dict, response_obj):
+    def _common_logic(self, kwargs: Mapping[str, object], response_obj):
         call_id: Final = response_obj.get("id", kwargs.get("litellm_call_id"))
         dt: Final = get_utc_datetime().isoformat()
         cost: Final = kwargs.get("response_cost", None)
@@ -74,8 +77,8 @@ class OpenMeterLogger(CustomLogger):
         # If no user provided directly, try to get it from token user_id
         if user_param is None:
             # Check if user_id is available from the API key metadata
-            litellm_params: Final = kwargs.get("litellm_params", {})
-            metadata: Final = litellm_params.get("metadata", {})
+            litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
+            metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
             user_api_key_user_id: Final = metadata.get("user_api_key_user_id", None)
 
             if user_api_key_user_id is not None:
@@ -89,7 +92,7 @@ class OpenMeterLogger(CustomLogger):
         return {
             "specversion": "1.0",
             "type": os.getenv("OPENMETER_EVENT_TYPE", "litellm_tokens"),
-            "id": call_id,
+            "id": logged_event_key(kwargs, call_id) if isinstance(call_id, str) else call_id,
             "time": dt,
             "subject": subject,
             "source": "litellm-proxy",

litellm/integrations/prometheus.py

@@ -3036,7 +3036,7 @@ class PrometheusLogger(CustomLogger):
 
     async def async_post_call_failure_hook(
         self,
-        request_data: dict,
+        request_data: dict[str, object],
         original_exception: Exception,
         user_api_key_dict: UserAPIKeyAuth,
         traceback_str: str | None = None,
@@ -3059,10 +3059,14 @@ class PrometheusLogger(CustomLogger):
         try:
             _tags: Final = StandardLoggingPayloadSetup.get_request_tags(
                 litellm_params=request_data,
-                proxy_server_request=request_data.get("proxy_server_request", {}),
+                proxy_server_request=_logged_fields(request_data.get("proxy_server_request")),
+            )
+            _metadata: Final = _logged_fields(request_data.get("metadata"))
+            model_id: Final = (
+                _logged_fields(_metadata.get("model_info")).get("id")
+                or _logged_fields(request_data.get("model_info")).get("id")
+                or _logged_fields(request_data.get("standard_logging_object")).get("model_id")
             )
-            _metadata: Final = request_data.get("metadata", {}) or {}
-            model_id: Final = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
             rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
             api_provider: Final = self._extract_api_provider_from_request_data(
                 request_data

litellm/litellm_core_utils/core_helpers.py

@@ -4,7 +4,7 @@ import copy
 import logging
 import math
 import re
-from collections.abc import Collection, Iterable, Iterator, Mapping
+from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
 
@@ -104,6 +104,10 @@ def as_str_mapping(value: object) -> Mapping[str, object] | None:
     return value if isinstance(value, Mapping) else None
 
 
+def as_object_list(value: object) -> Sequence[object] | None:
+    return value if isinstance(value, list) else None
+
+
 def decoded_json_object(value: object) -> Mapping[str, object]:
     mapping: Final = as_str_mapping(value)
     if mapping is not None:

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -107,6 +107,7 @@ class ExceptionCheckers:
             "exceeds the available context size",  # llama.cpp/Lemonade
             "exceeds the maximum number of tokens allowed",  # Gemini
             '"code": "context_length_exceeded"',
+            "prompt is too long",
         ]
         for substring in known_exception_substrings:
             if substring in _error_str_lowercase:

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -146,6 +146,10 @@ def handle_anthropic_text_model_custom_llm_provider(
     return model, custom_llm_provider
 
 
+def declared_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
+    return custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
+
+
 def declared_authenticating_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
     """The authenticating provider this pair already names, or None.
 
@@ -154,7 +158,7 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str
     and for a declared pair the resolver's answer is the declaration itself, so metadata callers
     adopt the declaration instead of resolving.
     """
-    declared: Final = custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
+    declared: Final = declared_provider(model, custom_llm_provider)
     return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None
 
 

litellm/litellm_core_utils/internal_call_metadata.py

@@ -26,7 +26,12 @@ from litellm.constants import (
     MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
     NON_INFERENCE_CALL_TYPES,
 )
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.initialize_dynamic_callback_params import initialize_standard_callback_dynamic_params
+from litellm.types.passthrough_endpoints.pass_through_endpoints import (
+    SETTLES_BACKGROUND_RESPONSE_KWARG,
+    BackgroundResponseSettlement,
+)
 from litellm.types.utils import (
     BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
     BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN,
@@ -35,6 +40,8 @@ from litellm.types.utils import (
 
 BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
 
+BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost"
+
 ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
 OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
 ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
@@ -151,6 +158,26 @@ def is_unbilled_non_inference_call_from_params(
     return is_unbilled_non_inference_call(call_type, metadata, response)
 
 
+def settles_background_response(call_type: str, metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> bool:
+    return kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES or (
+        call_type in NON_INFERENCE_CALL_TYPES
+        and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
+    )
+
+
+def logged_event_key(kwargs: Mapping[str, object], event_id: str) -> str:
+    call_type: Final = kwargs.get("call_type")
+    litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
+    metadata: Final = (
+        as_str_mapping(litellm_params.get("litellm_metadata")) or as_str_mapping(litellm_params.get("metadata")) or {}
+    )
+    if not event_id or not settles_background_response(
+        call_type if isinstance(call_type, str) else "", metadata, kwargs
+    ):
+        return event_id
+    return f"{event_id}{BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX}"
+
+
 def sanitize_user_api_key_auth(auth: object) -> object:
     """Copy of the auth object with its budget reservation removed; the cost callback
     falls back to reading the reservation from inside the auth object."""

litellm/litellm_core_utils/litellm_logging.py

@@ -872,6 +872,7 @@ class Logging(LiteLLMLoggingBaseClass):
         # The model the proxy shows the client on streamed chunks. The logged streamed response carries it
         # once that response is priced, the same way a non-streamed response is logged
         self.client_facing_stream_model: str | None = None
+        self.client_facing_response_id: str | None = None
         self.zero_cost_warned: bool = False
         self.llm_caching_handler: LLMCachingHandler | None = None
 
@@ -918,6 +919,7 @@ class Logging(LiteLLMLoggingBaseClass):
         # post_call guardrails have run; the @client decorator then stores the
         # enqueue closure here instead of firing it immediately.
         self.defer_async_logging: bool = False
+        self.drops_stream_token_ids: bool = False
         self.enqueue_deferred_logging: Callable[[], None] | None = None
         self._async_success_scheduled: bool = False
         self._async_success_enqueued: bool = False
@@ -3237,6 +3239,23 @@ class Logging(LiteLLMLoggingBaseClass):
             return response
         return response.model_copy(update={"id": response_id})
 
+    def _with_client_facing_response_id(
+        self,
+        response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
+    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
+        response_id: Final = self.client_facing_response_id
+        if response_id is None or getattr(response, "id", None) == response_id:
+            return response
+        return response.model_copy(update={"id": response_id})
+
+    def _client_facing_streaming_response(
+        self,
+        response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
+    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
+        return self._with_client_facing_response_id(
+            self._with_continued_response_id(self._with_client_facing_stream_model(response))
+        )
+
     def _success_handler_helper_fn(
         self,
         result: object = None,
@@ -3640,9 +3659,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 )
                 self._add_continued_partial_spend(complete_streaming_response)
                 self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
-                logged_streaming_response: Final = self._with_continued_response_id(
-                    self._with_client_facing_stream_model(complete_streaming_response)
-                )
+                logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
                 self.model_call_details["complete_streaming_response"] = logged_streaming_response
                 self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
                 ## STANDARDIZED LOGGING PAYLOAD
@@ -4239,9 +4256,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
             await self._prepare_baseline_cache_estimate(complete_streaming_response)
 
-            logged_streaming_response: Final = self._with_continued_response_id(
-                self._with_client_facing_stream_model(complete_streaming_response)
-            )
+            logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
             self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
 
             ## STANDARDIZED LOGGING PAYLOAD

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -3,7 +3,8 @@ import json
 import re
 import time
 import traceback
-from collections.abc import Generator, Iterator, Mapping, Sequence
+from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence
+from types import MappingProxyType
 from typing import Final, Literal, cast
 
 from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
@@ -11,7 +12,12 @@ from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
+from litellm.constants import (
+    RESPONSE_FORMAT_TOOL_NAME,
+    STREAM_CHUNK_CHOICE_FIELDS,
+    STREAM_FINISH_CHOICE_FIELDS,
+    VLLM_CHOICE_FIELDS,
+)
 from litellm.litellm_core_utils.hidden_params import get_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_reasoning_content,
@@ -170,6 +176,28 @@ def _split_assembled_content_for_replay(content: str | None) -> list[str]:
     return _REPLAY_CONTENT_SLICE_RE.findall(content)
 
 
+def _drop_choice_fields(choice: StreamingChoices, fields: Iterable[str]) -> None:
+    for field in fields:
+        if hasattr(choice, field):
+            delattr(choice, field)
+
+
+def _vllm_choice_fields(choice: Mapping[str, object]) -> Mapping[str, object]:
+    cached_fields: Final = _choice_provider_specific_fields(choice)
+    return MappingProxyType(
+        {
+            field: choice[field] if field in choice else cached_fields[field]
+            for field in VLLM_CHOICE_FIELDS
+            if field in choice or field in cached_fields
+        }
+    )
+
+
+def _choice_provider_specific_fields(choice: Mapping[str, object]) -> Mapping[str, object]:
+    fields: Final = choice.get("provider_specific_fields")
+    return _FIELD_MAP.validate_python(fields) if isinstance(fields, dict) else MappingProxyType({})
+
+
 _REPLAY_REASONING_FIELDS: Final = frozenset({"reasoning_content", "thinking_blocks", "reasoning_items"})
 _REPLAY_LEAD_FIELDS: Final = _REPLAY_REASONING_FIELDS | {"role", "provider_specific_fields"}
 _REPLAY_TOOL_FIELDS: Final = frozenset({"tool_calls", "function_call"})
@@ -220,8 +248,9 @@ def _replay_frame(
     choice.finish_reason = finish_reason
     if not keeps_choice_metadata:
         choice.logprobs = None
-        if hasattr(choice, "enhancements"):
-            del choice.enhancements
+        _drop_choice_fields(choice, ("enhancements", *STREAM_CHUNK_CHOICE_FIELDS))
+    if finish_reason is None:
+        _drop_choice_fields(choice, STREAM_FINISH_CHOICE_FIELDS)
     return frame
 
 
@@ -325,7 +354,9 @@ async def convert_to_streaming_response_async(
 
         logprobs = choice.get("logprobs", None)
 
-        choice = StreamingChoices(finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs)
+        choice = StreamingChoices(
+            finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs, **_vllm_choice_fields(choice)
+        )
         choice_list.append(choice)
 
     model_response_object.choices = choice_list
@@ -388,6 +419,7 @@ def convert_to_streaming_response(
             delta=delta,
             logprobs=logprobs,
             enhancements=enhancements,
+            **_vllm_choice_fields(choice),
         )
 
         choice_list.append(choice)
@@ -872,6 +904,7 @@ def convert_to_model_response_object(
                     logprobs=logprobs,
                     enhancements=enhancements,
                     provider_specific_fields=provider_specific_fields,
+                    **_vllm_choice_fields(choice),
                 )
                 if provider_sent_logprobs and logprobs is None:
                     choice.logprobs = None

litellm/litellm_core_utils/prompt_templates/chat_template_tools.py

@@ -10,12 +10,18 @@ class ChatTemplateToolSection:
     preamble_tokens: int
     implicit_system_turn_tokens: int
     render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
+    counts_tool_choice: bool = False
+    renders_strict_only_when_true: bool = False
 
 
-def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
+def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
     return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions)
 
 
+def _compact_json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
+    return "\n".join(json.dumps(function, ensure_ascii=False, separators=(",", ":")) for function in functions)
+
+
 def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
     return "".join(
         "\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions
@@ -23,13 +29,29 @@ def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
 
 
 _TOOL_SECTIONS: Final = (
+    (
+        re.compile(r"deepseek-v4(?:[-._]|p)1(?!\d)", re.IGNORECASE),
+        ChatTemplateToolSection(211, 0, _json_line_tool_schemas),
+    ),
     (
         re.compile(
             r"deepseek-v4(?![.\d]|p\d)|^(?:deepseek/)?deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE
         ),
-        ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas),
+        ChatTemplateToolSection(222, 0, _json_line_tool_schemas),
     ),
     (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
+    (
+        re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE),
+        ChatTemplateToolSection(
+            0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
+        ),
+    ),
+    (
+        re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE),
+        ChatTemplateToolSection(
+            0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
+        ),
+    ),
 )
 
 

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -29,6 +29,7 @@ from litellm.types.llms.openai import (
     ChatCompletionAudioObject,
     ChatCompletionFileObject,
     ChatCompletionImageObject,
+    ChatCompletionPromptCacheBreakpoint,
     ChatCompletionReasoningItem,
     ChatCompletionReasoningSummaryTextBlock,
     ChatCompletionRedactedThinkingBlock,
@@ -2193,16 +2194,62 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
 
 
 _PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
+_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(ChatCompletionPromptCacheBreakpoint)
+_AcceptedBreakpointT: Final = TypeVar("_AcceptedBreakpointT", bound=Mapping[str, object])
+
+
+def _accepted_prompt_cache_breakpoint(
+    accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
+) -> _AcceptedBreakpointT | None:
+    try:
+        return accepted.validate_python(marker)
+    except ValidationError:
+        return None
+
+
+def prompt_cache_breakpoint_within(
+    accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
+) -> _AcceptedBreakpointT | None:
+    """The client's marker cut down to the ``accepted`` shape: an unsupported ttl is dropped and the mode kept,
+    and a marker whose mode the target cannot read is dropped whole."""
+    whole: Final = _accepted_prompt_cache_breakpoint(accepted, marker)
+    if whole is not None:
+        return whole
+    fields: Final = _breakpoint_fields_or_none(marker)
+    without_ttl: Final = (
+        None
+        if fields is None
+        else _accepted_prompt_cache_breakpoint(accepted, {key: item for key, item in fields.items() if key != "ttl"})
+    )
+    verbose_logger.debug(
+        "Dropping malformed prompt_cache_breakpoint %r under drop_params"
+        if without_ttl is None
+        else "Dropping the unsupported ttl of prompt_cache_breakpoint %r under drop_params",
+        marker,
+    )
+    return without_ttl
 
 
 def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
     if marker is None or not drop_params:
         return marker
-    try:
-        return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
-    except ValidationError:
-        verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
-        return None
+    return prompt_cache_breakpoint_within(_PROMPT_CACHE_BREAKPOINT, marker)
+
+
+def prompt_cache_breakpoint_with_readable_mode(marker: object, drop_params: bool) -> object:
+    """The client's marker as written, or None under drop_params when no target could read its mode, so the
+    target's own config decides which ttl and keys it keeps."""
+    return None if prompt_cache_breakpoint_for_wire(marker, drop_params) is None else marker
+
+
+def whole_prompt_cache_breakpoint_or_none(marker: object) -> PromptCacheBreakpoint | None:
+    return _accepted_prompt_cache_breakpoint(_PROMPT_CACHE_BREAKPOINT, marker)
+
+
+def chat_completion_prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
+    if marker is None or not drop_params:
+        return marker
+    return prompt_cache_breakpoint_within(_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT, marker)
 
 
 def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
@@ -2220,6 +2267,13 @@ _BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object])
 _BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object])
 
 
+def _breakpoint_fields_or_none(value: object) -> Mapping[str, object] | None:
+    try:
+        return _BREAKPOINT_FIELDS.validate_python(value)
+    except ValidationError:
+        return None
+
+
 def _without_prompt_cache_breakpoint(value: object) -> object:
     if not isinstance(value, Mapping):
         return value
@@ -2242,6 +2296,27 @@ def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT:
     return cast(_MarkedT, stripped)  # cast-ok: same block shape as the input minus the marker keys
 
 
+def _part_with_chat_completion_prompt_cache_breakpoint(part: object) -> object:
+    if not isinstance(part, Mapping):
+        return part
+    fields: Final = _BREAKPOINT_FIELDS.validate_python(part)
+    marker: Final = fields.get("prompt_cache_breakpoint")
+    if marker is None:
+        return fields
+    unmarked: Final = {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"}
+    return with_prompt_cache_breakpoint(unmarked, chat_completion_prompt_cache_breakpoint_for_wire(marker, True))
+
+
+def chat_completion_prompt_cache_breakpoints_for_wire(message: _MarkedT, drop_params: bool) -> _MarkedT:
+    content: Final = message.get("content")
+    if not drop_params or not isinstance(content, list):
+        return message
+    parts: Final = [
+        _part_with_chat_completion_prompt_cache_breakpoint(part) for part in _BREAKPOINT_BLOCKS.validate_python(content)
+    ]
+    return cast(_MarkedT, {**message, "content": parts})  # cast-ok: same message shape with its parts' markers trimmed
+
+
 LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS: Final = frozenset(
     {"thinking_blocks", "reasoning_items", "provider_specific_fields"}
 )

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -34,6 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     split_leading_system_run,
     string_field,
 )
+from litellm.litellm_core_utils.tool_fields import tools_without_hosted_mcp
 from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
 from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value
 from litellm.types.files import get_file_extension_from_mime_type
@@ -1328,7 +1329,7 @@ def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
 _get_thought_signature_from_tool = get_thought_signature_from_tool
 
 
-def _get_dummy_thought_signature() -> str:
+def get_dummy_thought_signature() -> str:
     """Generate a dummy thought signature for models that require it.
 
     This is used when transferring conversation history from older models
@@ -1415,7 +1416,7 @@ def convert_to_gemini_tool_call_invoke(
                         # https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures#parallel_function_calling_example
                         is_first_function_call = len(_parts_list) == 0
                         if not thought_signature and is_first_function_call and needs_dummy_signature:
-                            thought_signature = _get_dummy_thought_signature()
+                            thought_signature = get_dummy_thought_signature()
                         if thought_signature:
                             part_dict["thoughtSignature"] = thought_signature
 
@@ -1439,7 +1440,7 @@ def convert_to_gemini_tool_call_invoke(
 
                 # If no signature found and model is gemini-3, use dummy signature
                 if not thought_signature and needs_dummy_signature:
-                    thought_signature = _get_dummy_thought_signature()
+                    thought_signature = get_dummy_thought_signature()
 
                 if thought_signature:
                     part_dict_function["thoughtSignature"] = thought_signature
@@ -5916,8 +5917,13 @@ def function_call_prompt(
     llm_provider: str,
     drop_params: bool,
 ) -> list[dict[str, object]]:
+    prompt_functions: Final = tools_without_hosted_mcp(
+        functions, model=model, route=llm_provider, drop_params=drop_params
+    )
+    if not prompt_functions:
+        return messages
     header: Final = """To call a function, reply with JSON ONLY in this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}}. Once a function result answers the request, reply to the user in plain text instead of calling a function again. The following functions are available to you:"""
-    function_prompt: Final = header + "".join(f"\n{function}\n" for function in functions)
+    function_prompt: Final = header + "".join(f"\n{function}\n" for function in prompt_functions)
     if not any(_is_system_message(message) for message in messages):
         return [*messages, {"role": "system", "content": function_prompt}]
     return [

litellm/litellm_core_utils/redact_messages.py

@@ -196,6 +196,9 @@ def _redact_function_call(function_call) -> None:
 
 def _redact_choice_content(choice):
     """Helper to redact content in a choice (message or delta)."""
+    if isinstance(choice, (litellm.Choices, litellm.utils.StreamingChoices, litellm.utils.TextChoices)):
+        drop_token_ids(choice)
+        _redact_provider_specific_fields(choice)
     if isinstance(choice, litellm.Choices):
         if choice.message.content is not None:
             choice.message.content = REDACTED_BY_LITELLM
@@ -216,6 +219,9 @@ def _redact_choice_content(choice):
         _redact_provider_specific_fields(choice.delta)
         _redact_tool_calls(getattr(choice.delta, "tool_calls", None))
         _redact_function_call(getattr(choice.delta, "function_call", None))
+    elif isinstance(choice, litellm.utils.TextChoices):
+        _redact_field_when_present(choice, "text")
+        _redact_field_when_present(choice, "reasoning_content")
 
 
 def _as_mapping(value: object) -> Mapping[str, object] | None:
@@ -227,14 +233,27 @@ def _redact_provider_specific_fields(message: object) -> None:
         setattr(message, "provider_specific_fields", _redacted_provider_specific_fields(fields))  # noqa: B010  # typed as object
 
 
+def _drop_provider_specific_token_ids(holder: object) -> None:
+    fields: Final = _as_mapping(getattr(holder, "provider_specific_fields", None))
+    if fields is not None and fields.get("token_ids") is not None:
+        setattr(holder, "provider_specific_fields", {**fields, "token_ids": None})  # noqa: B010  # typed as object
+
+
+def drop_token_ids(choice: object) -> None:
+    if getattr(choice, "token_ids", None) is not None:
+        setattr(choice, "token_ids", None)  # noqa: B010  # pydantic extra, not a declared field
+    for holder in (choice, getattr(choice, "delta", None), getattr(choice, "message", None)):
+        _drop_provider_specific_token_ids(holder)
+
+
 def _redact_provider_specific_fields_dict(choice: Mapping[str, object]) -> None:
-    for message in (choice.get("message"), choice.get("delta")):
+    for holder in (choice, choice.get("message"), choice.get("delta")):
         if (
-            (readable := _as_mapping(message)) is not None
+            (readable := _as_mapping(holder)) is not None
             and (fields := _as_mapping(readable.get("provider_specific_fields"))) is not None
-            and isinstance(message, dict)
+            and isinstance(holder, dict)
         ):
-            message["provider_specific_fields"] = _redacted_provider_specific_fields(fields)
+            holder["provider_specific_fields"] = _redacted_provider_specific_fields(fields)
 
 
 def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[str, object]:
@@ -251,6 +270,7 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
                 "compaction_blocks",
                 "compaction_start",
                 "compaction_delta",
+                "token_ids",
             )
             if key in fields
         },
@@ -422,6 +442,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
     for choice in choices:
         if isinstance(choice, dict):
             _redact_provider_specific_fields_dict(choice)
+            if choice.get("token_ids") is not None:
+                choice["token_ids"] = None
             if "message" in choice and isinstance(choice["message"], dict):
                 if choice["message"].get("content") is not None:
                     choice["message"]["content"] = redacted_str

litellm/litellm_core_utils/stream_usage_chunks.py

@@ -1,6 +1,7 @@
 from collections.abc import Iterable
 from typing import Final
 
+from litellm.constants import VLLM_CHOICE_FIELDS
 from litellm.types.utils import ModelResponseStream, StreamingChoices
 
 
@@ -9,12 +10,19 @@ def is_empty_streaming_choice(choice: StreamingChoices) -> bool:
         return False
     if getattr(choice, "logprobs", None) is not None:
         return False
+    if not _all_values_none(_vllm_choice_values(choice)):
+        return False
     delta: Final = choice.delta if hasattr(choice, "delta") else None
     if delta is None:
         return True
     return _all_values_none(delta.model_dump().values())
 
 
+def _vllm_choice_values(choice: StreamingChoices) -> Iterable[object]:
+    choice_extras: Final = choice.model_extra or {}
+    return (choice_extras.get(field) for field in VLLM_CHOICE_FIELDS)
+
+
 def _all_values_none(values: Iterable[object]) -> bool:
     return all(value is None for value in values)
 

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, Required
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.constants import VLLM_CHOICE_FIELDS
 from litellm.litellm_core_utils.core_helpers import (
     PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
     UPSTREAM_USAGE_COST,
@@ -31,6 +32,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_content_from_model_response,  # pyright: ignore[reportUnknownVariableType]  # response param accepts an untyped dict
 )
+from litellm.litellm_core_utils.redact_messages import drop_token_ids
 from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
 from litellm.types.llms.openai import (
     AllMessageValues,
@@ -434,6 +436,8 @@ class ChunkProcessor:
                 chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop"
             ),
             message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""},
+            provider_specific_fields=None,
+            **_streamed_vllm_choice_fields(chain.from_iterable(chunk.get("choices") or () for chunk in chunks)),
         )
 
     @staticmethod
@@ -504,6 +508,7 @@ class ChunkProcessor:
                             "index": 0,
                             "message": {"role": role, "content": ""},
                             "finish_reason": finish_reason,
+                            **_streamed_vllm_choice_fields(_first_choices(chunks)),
                         }
                     ]
                 ),
@@ -1371,6 +1376,7 @@ def _rebuilt_text_choice(index: int, choices: Sequence[TextChoices]) -> TextChoi
         index=index,
         logprobs=_merged_logprobs(choices),
         finish_reason=next((choice.finish_reason for choice in reversed(choices) if choice.finish_reason), None),
+        **_streamed_vllm_choice_fields(choices),
     )
 
 
@@ -1504,6 +1510,31 @@ def _field_of(value: object, name: str) -> object:
     return getattr(value, name, None)
 
 
+def _first_choices(chunks: Iterable[object]) -> tuple[object, ...]:
+    return tuple(next(iter(object_items(_field_of(chunk, "choices"))), None) for chunk in chunks)
+
+
+def _choice_extras(choice: object) -> Mapping[str, object]:
+    if isinstance(choice, BaseModel):
+        return choice.model_extra or {}
+    return object_mapping(choice) or {}
+
+
+def _streamed_vllm_choice_fields(choices: Iterable[object]) -> Mapping[str, object]:
+    extras: Final = tuple(map(_choice_extras, choices))
+    token_id_runs: Final = tuple(object_items(fields.get("token_ids")) for fields in extras)
+    stop_reasons: Final = tuple(reason for fields in extras if (reason := fields.get("stop_reason")) is not None)
+    has_token_ids: Final = any(fields.get("token_ids") is not None for fields in extras)
+    streamed_values: Final[Mapping[str, object]] = MappingProxyType(
+        {
+            "token_ids": list(chain.from_iterable(token_id_runs)) if has_token_ids else None,
+            "stop_reason": stop_reasons[-1] if stop_reasons else None,
+        }
+    )
+    present_fields: Final = frozenset(chain.from_iterable(fields.keys() for fields in extras))
+    return MappingProxyType({field: streamed_values[field] for field in VLLM_CHOICE_FIELDS if field in present_fields})
+
+
 def _delta_reasoning_items(delta: object) -> Sequence[object]:
     items: Final = _field_of(delta, "reasoning_items")
     return cast(Sequence[object], items) if isinstance(items, list) else ()  # cast-ok: items forwarded verbatim
@@ -1776,6 +1807,19 @@ def stream_chunk_builder(
     end_time: object = None,
     logging_obj: Optional["Logging"] = None,
     count_prompt_tokens: Callable[[], int] | None = None,
+) -> ModelResponse | TextCompletionResponse | None:
+    response: Final = _assembled_stream_response(chunks, messages, logging_obj, count_prompt_tokens)
+    if response is not None and getattr(logging_obj, "drops_stream_token_ids", False) is True:
+        for choice in response.choices:
+            drop_token_ids(choice)
+    return response
+
+
+def _assembled_stream_response(
+    chunks: Sequence[object],
+    messages: Sequence[AllMessageValues | Message] | None,
+    logging_obj: Optional["Logging"],
+    count_prompt_tokens: Callable[[], int] | None,
 ) -> ModelResponse | TextCompletionResponse | None:
     try:
         if chunks is None:
@@ -1788,7 +1832,7 @@ def stream_chunk_builder(
         if not chunks:
             return None
 
-        processor: Final = ChunkProcessor(chunks, messages)
+        processor: Final = ChunkProcessor(list(chunks), messages)
         chunks = processor.chunks
 
         ### BASE-CASE ###

litellm/litellm_core_utils/streaming_handler.py

@@ -23,6 +23,7 @@ from typing_extensions import NotRequired, TypedDict
 import litellm
 from litellm import verbose_logger
 from litellm._uuid import uuid
+from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_FIELDS
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.core_helpers import (
     UPSTREAM_USAGE_COST,
@@ -37,10 +38,11 @@ from litellm.litellm_core_utils.hidden_params import (
     set_hidden_param,
     stored_attribute,
 )
+from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_items
 from litellm.litellm_core_utils.model_response_utils import (
     is_model_response_stream_empty,
 )
-from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
+from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject, drop_token_ids
 from litellm.litellm_core_utils.thread_pool_executor import executor
 from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
 from litellm.types.llms.base import LiteLLMBaseModel
@@ -137,6 +139,70 @@ def _prepend_upstream_choice_extras(
     choice.__pydantic_fields_set__.update(upstream_extras)
 
 
+_NO_UPSTREAM_CHOICE_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+_NO_CHOICE_FIELDS_BY_INDEX: Final[Mapping[int, Mapping[str, object]]] = MappingProxyType({})
+
+
+def _upstream_first_choice(upstream_chunk: object) -> BaseModel | None:
+    match getattr(upstream_chunk, "choices", None):
+        case [BaseModel() as first_choice, *_]:
+            return first_choice
+        case _:
+            return None
+
+
+def _upstream_choice_fields(upstream_chunk: object, fields: Sequence[str]) -> Mapping[str, object]:
+    first_choice: Final = _upstream_first_choice(upstream_chunk)
+    if first_choice is None:
+        return _NO_UPSTREAM_CHOICE_FIELDS
+    upstream_extras: Final[Mapping[str, object]] = first_choice.model_extra or {}
+    return MappingProxyType({field: upstream_extras[field] for field in fields if field in upstream_extras})
+
+
+def _upstream_choice_index(upstream_chunk: object) -> int:
+    upstream_index: Final[object] = getattr(_upstream_first_choice(upstream_chunk), "index", 0)
+    return upstream_index if isinstance(upstream_index, int) else 0
+
+
+def _response_choice_index(response_obj: Mapping[str, object]) -> int:
+    choice_index: Final = response_obj.get("choice_index")
+    if isinstance(choice_index, int):
+        return choice_index
+    return _upstream_choice_index(response_obj.get("original_chunk"))
+
+
+def _non_null_upstream_choice_fields(upstream_chunk: object, fields: Sequence[str]) -> Mapping[str, object]:
+    first_choice: Final = _upstream_first_choice(upstream_chunk)
+    upstream_extras: Final = first_choice.model_extra if first_choice is not None else None
+    if not upstream_extras:
+        return _NO_UPSTREAM_CHOICE_FIELDS
+    return MappingProxyType(
+        {field: upstream_extras[field] for field in fields if upstream_extras.get(field) is not None}
+    )
+
+
+def _apply_choice_fields(choice: StreamingChoices, fields: Mapping[str, object]) -> None:
+    for field, value in fields.items():
+        setattr(choice, field, value)
+
+
+def _choice_fields_chunk(
+    model_response: ModelResponseStream, upstream_index: int, fields: Mapping[str, object]
+) -> ModelResponseStream:
+    model_response.choices[0].delta = Delta(content="")  # rebind-ok: chunks are built in place here
+    model_response.choices[0].index = upstream_index  # rebind-ok: chunks are built in place here
+    _apply_choice_fields(model_response.choices[0], fields)
+    return model_response
+
+
+def _without_token_ids_under_post_call_rules(chunk: ModelResponseStream | None) -> ModelResponseStream | None:
+    if chunk is None or not litellm.post_call_rules:
+        return chunk
+    for choice in chunk.choices:
+        drop_token_ids(choice)
+    return chunk
+
+
 def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool:
     return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields)
 
@@ -378,6 +444,8 @@ class CustomStreamWrapper:
         self._provider_response_model: str | None = None
         self._upstream_provider: str | None = None
         self.received_finish_reason: str | None = None
+        self.received_finish_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
+        self.carried_chunk_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
         self.intermittent_finish_reason: str | None = None  # finish reasons that show up mid-stream
         self.received_any_finish_reason = False  # a later chunk can reset intermittent_finish_reason
         self.special_tokens = [
@@ -823,6 +891,7 @@ class CustomStreamWrapper:
                 "logprobs": logprobs,
                 "original_chunk": str_line,
                 "usage": usage,
+                "chunk_choice_fields": _non_null_upstream_choice_fields(str_line, STREAM_CHUNK_CHOICE_FIELDS),
             }
         except Exception as e:
             raise e
@@ -847,7 +916,7 @@ class CustomStreamWrapper:
         except Exception as e:
             raise e
 
-    def handle_openai_text_completion_chunk(self, chunk):
+    def handle_openai_text_completion_chunk(self, chunk: object):
         try:
             text = ""
             is_finished = False
@@ -860,11 +929,15 @@ class CustomStreamWrapper:
                     is_finished = True
                     finish_reason = choices[0].finish_reason
             usage = getattr(chunk, "usage", None)
+            choice_index: Final = _upstream_choice_index(chunk)
             return {
                 "text": text,
                 "is_finished": is_finished,
                 "finish_reason": finish_reason,
                 "usage": usage,
+                "chunk_choice_fields": _non_null_upstream_choice_fields(chunk, STREAM_CHUNK_CHOICE_FIELDS),
+                "choice_index": choice_index,
+                "sibling_finish_reason": finish_reason if choice_index != 0 else None,
             }
 
         except Exception as e:
@@ -1183,6 +1256,13 @@ class CustomStreamWrapper:
         )
 
         is_chunk_non_empty: Final = self.is_chunk_non_empty(completion_obj, model_response, response_obj)
+        upstream_chunk_choice_fields: Final[Mapping[str, object]] = (
+            _NO_UPSTREAM_CHOICE_FIELDS
+            if litellm.post_call_rules
+            else response_obj.get("chunk_choice_fields", _NO_UPSTREAM_CHOICE_FIELDS)
+        )
+        chunk_choice_fields: Final = self._take_carried_chunk_choice_fields(response_obj, upstream_chunk_choice_fields)
+        sibling_finish_reason: Final[str | None] = response_obj.get("sibling_finish_reason")
 
         if is_chunk_non_empty:  # cannot set content of an OpenAI Object to be an empty string
             self.raise_on_model_repetition()
@@ -1270,16 +1350,40 @@ class CustomStreamWrapper:
                     if response_obj.get("provider_specific_fields") is not None:
                         completion_obj["provider_specific_fields"] = response_obj["provider_specific_fields"]
                     model_response.choices[0].delta = Delta(**completion_obj)
-                    _index: Final[int | None] = completion_obj.get("index")
+                    _index: Final[int | None] = completion_obj.get("index", response_obj.get("choice_index"))
                     if _index is not None:
                         model_response.choices[0].index = _index
 
+                _apply_choice_fields(model_response.choices[0], chunk_choice_fields)
+                if sibling_finish_reason is not None:
+                    self._finish_sibling_choice(model_response.choices[0], sibling_finish_reason)
+                elif response_obj.get("is_finished"):
+                    self._apply_finish_choice_fields(model_response.choices[0])
                 self._optional_combine_thinking_block_in_choices(model_response=model_response)
 
                 return model_response
             else:
                 return
+        elif sibling_finish_reason is not None:
+            sibling_chunk: Final = _choice_fields_chunk(
+                model_response, _response_choice_index(response_obj), chunk_choice_fields
+            )
+            self._finish_sibling_choice(sibling_chunk.choices[0], sibling_finish_reason)
+            return sibling_chunk
         elif self.received_finish_reason is not None:
+            chunk_upstream_index: Final = _response_choice_index(response_obj)
+            if chunk_choice_fields and self._streams_past_the_pending_finish(
+                model_response, response_obj, chunk_upstream_index
+            ):
+                self.carried_chunk_choice_fields = MappingProxyType(
+                    {**self.carried_chunk_choice_fields, chunk_upstream_index: chunk_choice_fields}
+                )
+            else:
+                another_choice_fields_chunk: Final = self._another_choice_fields_chunk(
+                    model_response, response_obj, chunk_choice_fields
+                )
+                if another_choice_fields_chunk is not None:
+                    return another_choice_fields_chunk
             if self.sent_last_chunk is True:
                 # Bedrock returns the guardrail trace in the last chunk - we want to return this here
                 if self.custom_llm_provider == "bedrock" and "trace" in model_response:
@@ -1317,12 +1421,20 @@ class CustomStreamWrapper:
                 model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
                     map_finish_reason(finish_reason=self.received_finish_reason)
                 )  # ensure consistent output to openai
+                self._apply_finish_choice_fields(model_response.choices[0])
+                if (
+                    response_obj.get("is_finished")
+                    and _response_choice_index(response_obj) == model_response.choices[0].index
+                ):
+                    _apply_choice_fields(model_response.choices[0], chunk_choice_fields)
 
                 self.sent_last_chunk = True
 
             return model_response
         elif self._has_special_delta_content(model_response):
             return self._handle_special_delta_content(model_response)
+        elif chunk_choice_fields:
+            return _choice_fields_chunk(model_response, _response_choice_index(response_obj), chunk_choice_fields)
         else:
             if hasattr(model_response, "usage"):
                 self.chunks.append(model_response)
@@ -1545,6 +1657,8 @@ class CustomStreamWrapper:
             completion_obj["content"] = response_obj["text"]
             print_verbose(f"completion obj content: {completion_obj['content']}")
             if response_obj["is_finished"]:
+                self._record_finish_choice_fields(chunk)
+            if response_obj["is_finished"] and response_obj["sibling_finish_reason"] is None:
                 self.received_finish_reason = response_obj["finish_reason"]
             if response_obj["usage"] is not None:
                 _text_completion_usage: Final[Usage] = response_obj["usage"]
@@ -1608,6 +1722,7 @@ class CustomStreamWrapper:
                 self.system_fingerprint = cached_chunk.system_fingerprint
             if response_obj["is_finished"]:
                 self.received_finish_reason = response_obj["finish_reason"]
+                self._record_finish_choice_fields(cached_chunk)
         else:  # openai / azure chat model
             self.uses_openai_chat_stream = True
             if isinstance(chunk, ChatCompletionChunk):
@@ -1637,6 +1752,7 @@ class CustomStreamWrapper:
                         f"{self.custom_llm_provider} raised a streaming error - finish_reason: error, no content string given. Received Chunk={response_obj}"
                     )
                 self.received_finish_reason = response_obj["finish_reason"]
+                self._record_finish_choice_fields(chunk)
             if response_obj.get("original_chunk", None) is not None:
                 if hasattr(response_obj["original_chunk"], "id"):
                     model_response = self.set_model_id(response_obj["original_chunk"].id, model_response)
@@ -1704,7 +1820,7 @@ class CustomStreamWrapper:
                 completion_obj=completion_obj,
             )
             if isinstance(dispatch_result, _ProviderChunkEarlyReturn):
-                return dispatch_result.value
+                return _without_token_ids_under_post_call_rules(dispatch_result.value)
             response_obj = dispatch_result.response_obj
 
             model_response.model = self._served_openrouter_model() or self.model
@@ -1794,7 +1910,7 @@ class CustomStreamWrapper:
                 response_obj=response_obj,
                 plain_delta_created=plain_delta_created,
             )
-            return result
+            return _without_token_ids_under_post_call_rules(result)
 
         except StopIteration:
             raise StopIteration
@@ -1987,6 +2103,75 @@ class CustomStreamWrapper:
             return "tool_calls"
         return finish_reason
 
+    def _record_finish_choice_fields(self, upstream_chunk: object) -> None:
+        self.received_finish_choice_fields = MappingProxyType(
+            {
+                **self.received_finish_choice_fields,
+                _upstream_choice_index(upstream_chunk): _upstream_choice_fields(
+                    upstream_chunk, STREAM_FINISH_CHOICE_FIELDS
+                ),
+            }
+        )
+
+    def _apply_finish_choice_fields(self, choice: StreamingChoices) -> None:
+        _apply_choice_fields(choice, self.received_finish_choice_fields.get(choice.index, _NO_UPSTREAM_CHOICE_FIELDS))
+
+    def _finish_sibling_choice(self, choice: StreamingChoices, finish_reason: str) -> None:
+        choice.finish_reason = map_finish_reason(finish_reason)  # rebind-ok: chunks are built in place here
+        self._apply_finish_choice_fields(choice)
+
+    def _streams_past_the_pending_finish(
+        self, model_response: ModelResponseStream, response_obj: Mapping[str, object], upstream_index: int
+    ) -> bool:
+        finishing_index: Final = model_response.choices[0].index
+        return (
+            not self.sent_last_chunk
+            and not response_obj.get("is_finished")
+            and upstream_index != finishing_index
+            and finishing_index in self.received_finish_choice_fields
+        )
+
+    def _take_carried_chunk_choice_fields(
+        self, response_obj: Mapping[str, object], chunk_choice_fields: Mapping[str, object]
+    ) -> Mapping[str, object]:
+        if not self.carried_chunk_choice_fields:
+            return chunk_choice_fields
+        upstream_index: Final = _response_choice_index(response_obj)
+        carried: Final = self.carried_chunk_choice_fields.get(upstream_index)
+        if carried is None:
+            return chunk_choice_fields
+        self.carried_chunk_choice_fields = MappingProxyType(
+            {index: fields for index, fields in self.carried_chunk_choice_fields.items() if index != upstream_index}
+        )
+        return MappingProxyType(
+            {
+                field: [*object_items(carried.get(field)), *object_items(chunk_choice_fields.get(field))]
+                for field in STREAM_CHUNK_CHOICE_FIELDS
+                if field in carried or field in chunk_choice_fields
+            }
+        )
+
+    def _another_choice_fields_chunk(
+        self,
+        model_response: ModelResponseStream,
+        response_obj: Mapping[str, object],
+        chunk_choice_fields: Mapping[str, object],
+    ) -> ModelResponseStream | None:
+        upstream_index: Final = _response_choice_index(response_obj)
+        is_finished: Final = bool(response_obj.get("is_finished"))
+        if is_finished and upstream_index == model_response.choices[0].index and not self.sent_last_chunk:
+            return None
+        finish_fields: Final = (
+            self.received_finish_choice_fields.get(upstream_index, _NO_UPSTREAM_CHOICE_FIELDS)
+            if is_finished
+            else _NO_UPSTREAM_CHOICE_FIELDS
+        )
+        if not chunk_choice_fields and all(value is None for value in finish_fields.values()):
+            return None
+        return _choice_fields_chunk(
+            model_response, upstream_index, MappingProxyType({**finish_fields, **chunk_choice_fields})
+        )
+
     def finish_reason_handler(self):
         model_response: Final = self.model_response_creator(
             hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
@@ -2005,6 +2190,7 @@ class CustomStreamWrapper:
         model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
             _finish_reason if _finish_reason is not None else "stop"
         )
+        self._apply_finish_choice_fields(model_response.choices[0])
         close_tag: Final = self._close_open_think_block()
         if close_tag is not None:
             model_response.choices[0].delta.content = close_tag

litellm/litellm_core_utils/token_counter.py

@@ -990,7 +990,11 @@ def _count_extra(
         tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known.
     """
     reply_primer_tokens: Final = 3
-    tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice)
+    tool_choice_tokens: Final = (
+        _count_tool_choice(count_function, tool_choice)
+        if tool_section is None or tool_section.counts_tool_choice
+        else 0
+    )
     return (
         reply_primer_tokens
         + _count_tool_definitions(count_function, tools, includes_system_message, tool_section)
@@ -1029,14 +1033,29 @@ def _count_tool_choice(
             return 0
 
 
+def _named_function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]:
+    return (function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name"))
+
+
+def sends_function_tools(tools: Sequence[object] | None) -> bool:
+    return tools is not None and any(_named_function_definitions(tools))
+
+
+def _without_false_strict(function: Mapping[str, object]) -> Mapping[str, object]:
+    return {key: value for key, value in function.items() if key != "strict" or value}
+
+
 def _count_chat_template_tools(
     count_function: TokenCounterFunction,
     tools: Sequence[object],
     includes_system_message: bool,
     tool_section: ChatTemplateToolSection,
 ) -> int:
-    functions: Final = tuple(
-        function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name")
+    named_functions: Final = tuple(_named_function_definitions(tools))
+    functions: Final = (
+        tuple(map(_without_false_strict, named_functions))
+        if tool_section.renders_strict_only_when_true
+        else named_functions
     )
     if not functions:
         return 0

litellm/litellm_core_utils/tool_fields.py

@@ -1,13 +1,16 @@
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Mapping, Sequence
+from copy import copy
 from dataclasses import dataclass
 from functools import reduce
-from typing import Annotated, Final, TypeAlias
+from typing import Annotated, Final, TypeAlias, TypeVar
 
 from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
 import litellm
-from litellm.litellm_core_utils.core_helpers import normalize_drop_params
+from litellm._logging import verbose_logger
+from litellm.exceptions import UnsupportedParamsError
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
 
 ROUTES_WITH_TOOLS: Final = frozenset(
     {
@@ -162,3 +165,145 @@ def resolve_deployment_tool_fields(
     if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
         return kwargs
     return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))
+
+
+_ToolT: Final = TypeVar("_ToolT")
+
+HOSTED_MCP_DROP_WARNING: Final = (
+    "Dropping the hosted MCP tool(s) %s for model=%s on %s (drop_params=True) because this route cannot run a "
+    "hosted MCP tool."
+)
+
+
+def _hosted_mcp_server_label(tool: object) -> str | None:
+    match tool:
+        case {"type": "mcp", "server_label": str() as server_label}:
+            return server_label
+        case {"type": "mcp"}:
+            return "unlabeled"
+        case _:
+            return None
+
+
+def _drop_params_hint(dropped: str) -> str:
+    return (
+        f"To drop {dropped} instead, set `litellm.drop_params=True` or for proxy:\n\n"
+        "`litellm_settings:\n drop_params: true`\n"
+    )
+
+
+def _one_log_line(text: str) -> str:
+    return text.replace("\r", "").replace("\n", "")
+
+
+def _unrunnable_mcp_error(message: str, model: str, route: str) -> UnsupportedParamsError:
+    return UnsupportedParamsError(message=message, model=model, llm_provider=route, status_code=400)
+
+
+def tools_without_hosted_mcp(tools: Sequence[_ToolT], model: str, route: str, drop_params: bool) -> Sequence[_ToolT]:
+    server_labels: Final = tuple(label for label in map(_hosted_mcp_server_label, tools) if label is not None)
+    if not server_labels:
+        return tools
+    named_servers: Final = ", ".join(f"`{label}`" for label in server_labels)
+    if not drops_unsupported_params(drop_params):
+        raise _unrunnable_mcp_error(
+            f'{model} on {route} cannot run the hosted MCP tool {named_servers} (`type: "mcp"`). To have LiteLLM '
+            "run the MCP server for this model, register it on the LiteLLM MCP gateway and send "
+            '`server_url: "litellm_proxy/mcp/<server name>"`. ' + _drop_params_hint("hosted MCP tools"),
+            model=model,
+            route=route,
+        )
+    verbose_logger.warning(
+        HOSTED_MCP_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route)
+    )
+    return [tool for tool in tools if _hosted_mcp_server_label(tool) is None]
+
+
+MCP_SERVERS_DROP_WARNING: Final = (
+    "Dropping mcp_servers %s and their mcp_toolset tools for model=%s on %s (drop_params=True) because LiteLLM "
+    "cannot pass mcp_servers to this route."
+)
+
+ROUTES_WITHOUT_MCP_SERVERS: Final = frozenset(
+    {"bedrock", "databricks", "deepseek", "gemini", "gigachat", "ollama", "ollama_chat", "vertex_ai", "vertex_ai_beta"}
+)
+
+
+def _mcp_server_name(server: object) -> str:
+    match server:
+        case {"name": str() as name}:
+            return name
+        case _:
+            return "unnamed"
+
+
+def _mcp_server_names(mcp_servers: object) -> tuple[str, ...]:
+    if not mcp_servers:
+        return ()
+    if not isinstance(mcp_servers, list):
+        return ("unnamed",)
+    return tuple(map(_mcp_server_name, _TOOLS.validate_python(mcp_servers)))
+
+
+def _is_mcp_toolset(tool: object) -> bool:
+    match tool:
+        case {"type": "mcp_toolset"}:
+            return True
+        case _:
+            return False
+
+
+def drops_mcp_servers(request_kwargs: Mapping[str, object]) -> bool:
+    additional_drop_params: Final = request_kwargs.get("additional_drop_params")
+    return drops_unsupported_params(request_kwargs.get("drop_params")) or (
+        isinstance(additional_drop_params, list) and "mcp_servers" in additional_drop_params
+    )
+
+
+def tools_without_mcp_servers(
+    mcp_servers: object, tools: Sequence[_ToolT], model: str, route: str, drop_params: bool
+) -> Sequence[_ToolT]:
+    server_names: Final = _mcp_server_names(mcp_servers)
+    if not server_names:
+        return tools
+    named_servers: Final = ", ".join(f"`{name}`" for name in server_names)
+    if not drops_unsupported_params(drop_params):
+        raise _unrunnable_mcp_error(
+            f"LiteLLM cannot pass `mcp_servers` to {model} on {route}, so the MCP server {named_servers} would not "
+            'run. To have LiteLLM run it for this model, register it on the LiteLLM MCP gateway and send a `type: "mcp"` '
+            'tool with `server_url: "litellm_proxy/mcp/<server name>"`. ' + _drop_params_hint("`mcp_servers`"),
+            model=model,
+            route=route,
+        )
+    verbose_logger.warning(
+        MCP_SERVERS_DROP_WARNING, _one_log_line(named_servers), _one_log_line(model), _one_log_line(route)
+    )
+    return [tool for tool in tools if not _is_mcp_toolset(tool)]
+
+
+_TOOL_SELECTION_PARAMS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls"})
+_ParamsT: Final = TypeVar("_ParamsT", bound=dict[str, object])
+
+
+def tool_params_without_hosted_mcp(
+    params: _ParamsT, model: str, route: str, drop_params: bool, runs_hosted_mcp: Callable[[], bool] | None = None
+) -> _ParamsT:
+    tools: Final = params.get("tools")
+    if not isinstance(tools, list) or not tools:
+        return params
+    sent_tools: Final = _TOOLS.validate_python(tools)
+    if all(_hosted_mcp_server_label(tool) is None for tool in sent_tools) or (
+        runs_hosted_mcp is not None and runs_hosted_mcp()
+    ):
+        return params
+    kept_tools: Final = tools_without_hosted_mcp(tools=sent_tools, model=model, route=route, drop_params=drop_params)
+    if len(kept_tools) == len(sent_tools):
+        return params
+    kept_params: Final = copy(params)
+    if kept_tools:
+        kept_params["tools"] = list(kept_tools)
+        return kept_params
+    dropped_keys: Final = ("tools",) if params.get("functions") else _TOOL_SELECTION_PARAMS
+    for key in dropped_keys:
+        _ = kept_params.pop(key, None)
+    return kept_params

litellm/llms/anthropic/chat/handler.py

@@ -516,6 +516,7 @@ class AnthropicChatCompletion(BaseLLM):
                 headers=headers,
                 request_data=request_data,
                 provider=custom_llm_provider,
+                api_base=api_base,
             )
             data: Final = dict(
                 strip_beta_gated_request_body(
@@ -1194,6 +1195,9 @@ class ModelResponseIterator:
                 start_message: Final = message_start_block["message"]
                 if "model" in start_message:
                     self.served_model = start_message["model"]
+                input_transformations: Final = start_message.get("input_transformations")
+                if input_transformations is not None:
+                    provider_specific_fields["input_transformations"] = input_transformations
                 if "usage" in message_start_block["message"]:
                     usage = self._handle_usage(anthropic_usage_chunk=message_start_block["message"]["usage"])
             elif type_chunk == "error":

litellm/llms/anthropic/chat/transformation.py

@@ -13,6 +13,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
 import litellm
+from litellm.anthropic_beta_headers_manager import get_unsupported_headers
 from litellm.constants import (
     ANTHROPIC_BILLING_METADATA_PREFIX,
     ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
@@ -56,6 +57,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     split_leading_system_run,
 )
 from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
+from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
 from litellm.llms.base_llm.base_utils import (
     ResponseFormatSchema,
     as_json_object,
@@ -1315,6 +1317,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             *(tool for tool in tools[first_search_tool_index + 1 :] if not _is_openai_tool_search_tool(tool)),
         ]
 
+    def runs_hosted_mcp(self) -> bool:
+        return ANTHROPIC_BETA_HEADER_VALUES.MCP_CLIENT_2025_04_04.value not in get_unsupported_headers(
+            self._resolved_provider
+        )
+
     def _tools_param_with_openai_tool_search_mapped(
         self, non_default_params: Mapping[str, object], model: str, drop_params: bool
     ) -> Mapping[str, object]:
@@ -1903,7 +1910,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         model: str,
         drop_params: bool,
     ) -> dict:
-        is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=non_default_params)
+        request_params: Final = tool_params_without_hosted_mcp(
+            non_default_params,
+            model=model,
+            route=self._resolved_provider,
+            drop_params=drop_params,
+            runs_hosted_mcp=self.runs_hosted_mcp,
+        )
+        is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=request_params)
 
         # NB: ``map_openai_params`` deliberately does NOT sanitize tool names
         # here. Names are the *original* OpenAI names at this stage, and must
@@ -1916,7 +1930,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         #      via ``data = {**optional_params}`` and would 400 with
         #      ``Extra inputs are not permitted``).
 
-        for param, value in non_default_params.items():
+        for param, value in request_params.items():
             if param == "max_tokens" or param == "max_completion_tokens":
                 optional_params["max_tokens"] = value if isinstance(value, int) else max(1, int(round(value)))
             elif param == "tools":
@@ -1930,8 +1944,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                     optional_params["mcp_servers"] = mcp_servers
             elif param == "tool_choice" or param == "parallel_tool_calls":
                 _tool_choice: AnthropicMessagesToolChoice | None = self._map_tool_choice(
-                    tool_choice=non_default_params.get("tool_choice"),
-                    parallel_tool_use=non_default_params.get("parallel_tool_calls"),
+                    tool_choice=request_params.get("tool_choice"),
+                    parallel_tool_use=request_params.get("parallel_tool_calls"),
                 )
 
                 if _tool_choice is not None:
@@ -1957,7 +1971,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 if self.uses_native_structured_output_for_schema(
                     model,
                     self._extract_json_schema_from_response_format(value),
-                    REQUEST_PARAMS.validate_python(non_default_params),
+                    REQUEST_PARAMS.validate_python(request_params),
                 ):
                     _output_format = self.map_response_format_to_anthropic_output_format(value)
                     if _output_format is not None:
@@ -1993,7 +2007,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                     # `thinking={type: enabled, budget_tokens}` interface a
                     # pre-4.6 model actually supports instead of forwarding a
                     # shape the model will reject.
-                    max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens")
+                    max_tokens = request_params.get("max_completion_tokens") or request_params.get("max_tokens")
                     legacy_thinking = AnthropicConfig._map_reasoning_effort(
                         reasoning_effort="medium",
                         model=model,
@@ -2087,7 +2101,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
         ## handle thinking tokens
         self.update_optional_params_with_thinking_tokens(
-            non_default_params=non_default_params, optional_params=optional_params
+            non_default_params=request_params, optional_params=optional_params
         )
 
         return optional_params
@@ -3028,7 +3042,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
     def _build_provider_specific_fields(
         self,
-        completion_response: dict,
+        completion_response: Mapping[str, object],
         citations: Sequence[object] | None,
         thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None,
         web_search_results: Sequence[object] | None,
@@ -3045,6 +3059,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if context_management is not None:
             provider_specific_fields["context_management"] = context_management
 
+        input_transformations: Final = completion_response.get("input_transformations")
+        if input_transformations is not None:
+            provider_specific_fields["input_transformations"] = input_transformations
+
         if web_search_results is not None:
             provider_specific_fields["web_search_results"] = web_search_results
 
@@ -3055,17 +3073,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if tool_results is not None:
             provider_specific_fields["tool_results"] = tool_results
 
+        container: Final = completion_response.get("container")
+        container_fields: Final[Mapping[str, object] | None] = container if isinstance(container, dict) else None
         if tool_results is not None and any(result.get("type") != "tool_search_tool_result" for result in tool_results):
-            container_id: Final = (
-                completion_response.get("container", {}).get("id")
-                if isinstance(completion_response.get("container"), dict)
-                else None
-            )
+            raw_container_id: Final = container_fields.get("id") if container_fields is not None else None
+            container_id: Final = raw_container_id if isinstance(raw_container_id, str) else None
             code_by_id: Final = self._build_code_by_id_map(tool_calls)
             code_interpreter_results = self._build_code_interpreter_results(tool_results, code_by_id, container_id)
             provider_specific_fields["code_interpreter_results"] = code_interpreter_results
 
-        container: Final = completion_response.get("container")
         if container is not None:
             provider_specific_fields["container"] = container
 
@@ -3166,8 +3182,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 STOP_REASON_FIELDS.validate_python(completion_response)
             )
             completion_response["stop_reason"] = "stop"
-            if stop_reason_fields:
-                json_mode_message.provider_specific_fields = dict(stop_reason_fields)
+            input_transformations: Final = provider_specific_fields.get("input_transformations")
+            json_mode_fields: Final = {
+                **stop_reason_fields,
+                **({} if input_transformations is None else {"input_transformations": input_transformations}),
+            }
+            if json_mode_fields:
+                json_mode_message.provider_specific_fields = json_mode_fields
             if "refusal" in refusal_fields:
                 json_mode_message.refusal = refusal_fields["refusal"]
             if thinking_blocks:

litellm/llms/anthropic/common_utils.py

@@ -1297,7 +1297,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             return None
         return anthropic_beta_header.split(",")
 
-    def get_computer_tool_beta_header(self, computer_tool_version: str) -> str:
+    def get_computer_tool_beta_header(self, computer_tool_version: str) -> str | None:
         """
         Get the appropriate beta header for a given computer tool version.
 
@@ -1305,16 +1305,14 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             computer_tool_version: The computer tool version (e.g., 'computer_20250124', 'computer_20241022')
 
         Returns:
-            The corresponding beta header string
+            The corresponding beta header string, or None for a version with no known beta
         """
         computer_tool_beta_mapping: Final = {
+            "computer_20251124": "computer-use-2025-11-24",
             "computer_20250124": "computer-use-2025-01-24",
             "computer_20241022": "computer-use-2024-10-22",
         }
-        return computer_tool_beta_mapping.get(
-            computer_tool_version,
-            "computer-use-2024-10-22",  # Default fallback
-        )
+        return computer_tool_beta_mapping.get(computer_tool_version)
 
     def get_anthropic_beta_list(
         self,
@@ -1346,9 +1344,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         if effort_used:
             betas.append(ANTHROPIC_EFFORT_BETA_HEADER)  # effort-2025-11-24
 
-        if computer_tool_used:
-            beta_header: Final = self.get_computer_tool_beta_header(computer_tool_used)
-            betas.append(beta_header)
+        if computer_tool_used and (computer_tool_beta := self.get_computer_tool_beta_header(computer_tool_used)):
+            betas.append(computer_tool_beta)
 
         # Anthropic no longer requires the prompt-caching beta header
         # Prompt caching now works automatically when cache_control is used in messages
@@ -1437,9 +1434,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         # Anthropic no longer requires the prompt-caching beta header
         # Prompt caching now works automatically when cache_control is used in messages
         # Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
-        if computer_tool_used:
-            beta_header: Final = self.get_computer_tool_beta_header(computer_tool_used)
-            betas.add(beta_header)
+        if computer_tool_used and (computer_tool_beta := self.get_computer_tool_beta_header(computer_tool_used)):
+            betas.add(computer_tool_beta)
         # if pdf_used:
         #     betas.add("pdfs-2024-09-25")
         if file_id_used:

litellm/llms/anthropic/messages/adapters/handler.py

@@ -15,6 +15,11 @@ from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
+from litellm.litellm_core_utils.tool_fields import (
+    ROUTES_WITHOUT_MCP_SERVERS,
+    drops_mcp_servers,
+    tools_without_mcp_servers,
+)
 from litellm.llms.anthropic.messages.adapters.transformation import (
     AnthropicAdapter,
 )
@@ -623,6 +628,23 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             Logging as LiteLLMLoggingObject,
         )
 
+        provider_kwargs: Final = extra_kwargs or {}
+        custom_llm_provider: Final = provider_kwargs.get("custom_llm_provider")
+        target_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
+        drop_params: Final = drops_unsupported_params(provider_kwargs.get("drop_params"))
+        strips_mcp_servers: Final = target_provider in ROUTES_WITHOUT_MCP_SERVERS
+        request_tools: Final = (
+            tools_without_mcp_servers(
+                provider_kwargs.get("mcp_servers"),
+                tools or (),
+                model=model,
+                route=target_provider,
+                drop_params=drops_mcp_servers(provider_kwargs),
+            )
+            if strips_mcp_servers
+            else tools
+        )
+
         request_data: Final[dict[str, object]] = {
             "model": model,
             "messages": messages,
@@ -639,10 +661,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             request_data["temperature"] = temperature
         if thinking:
             request_data["thinking"] = thinking
-        if tool_choice:
+        if tool_choice and (request_tools or not tools):
             request_data["tool_choice"] = tool_choice
-        if tools:
-            request_data["tools"] = tools
+        if request_tools:
+            request_data["tools"] = request_tools
         if top_k is not None:
             request_data["top_k"] = top_k
         if top_p is not None:
@@ -660,16 +682,15 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         if "output_config" in extra_kwargs:
             request_data["output_config"] = extra_kwargs["output_config"]
 
-        custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
         api_base: Final = extra_kwargs.get("api_base")
         (
             openai_request,
             tool_name_mapping,
         ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
             request_data,
-            custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
+            custom_llm_provider=target_provider,
             api_base=api_base if isinstance(api_base, str) else None,
-            drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
+            drop_params=drop_params,
         )
 
         if openai_request is None:
@@ -697,7 +718,11 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         # Maintainability: when adding a new Anthropic-only request param to
         # ``AnthropicMessagesRequestOptionalParams``, also extend
         # ``ANTHROPIC_ONLY_REQUEST_KEYS`` here so it doesn't silently leak.
-        excluded_keys: Final = ANTHROPIC_ONLY_REQUEST_KEYS | CONVERTED_STREAM_KEYS | {"anthropic_messages"}
+        excluded_keys: Final = (
+            ANTHROPIC_ONLY_REQUEST_KEYS
+            | CONVERTED_STREAM_KEYS
+            | ({"anthropic_messages", "mcp_servers"} if strips_mcp_servers else {"anthropic_messages"})
+        )
         # NOTE: extra_kwargs was already coerced from None to {} at the top of
         # this method (line ~220). It is guaranteed to be a dict here.
         for key, value in extra_kwargs.items():

litellm/llms/anthropic/messages/adapters/streaming_iterator.py

@@ -493,10 +493,23 @@ class _CombinedChunkSplitter:
             for position, block in enumerate(blocks)
         )
 
+    @staticmethod
+    def _without_sibling_finish(chunk: "ModelResponseStream") -> "ModelResponseStream":
+        choices: Final = _optional_attr_sequence(chunk, "choices")
+        if (
+            len(choices) != 1
+            or _optional_attr(choices[0], "index") in (None, 0)
+            or _optional_attr(choices[0], "finish_reason") is None
+        ):
+            return chunk
+        unfinished: Final = copy.deepcopy(chunk)
+        unfinished.choices[0].finish_reason = None
+        return unfinished
+
     def _expand(self, chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
         split_chunks: Final = tuple(
             split_chunk
-            for combined_chunk in self._split(chunk)
+            for combined_chunk in self._split(self._without_sibling_finish(chunk))
             for split_chunk in self._split_by_payload_kind(combined_chunk)  # comprehension-ok: flattens nested splits
         )
         block_chunks: Final = tuple(itertools.chain.from_iterable(map(self._split_thinking_blocks, split_chunks)))

litellm/llms/anthropic/messages/context_management/inference_view.py

@@ -0,0 +1,138 @@
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Final
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+from typing_extensions import assert_never
+
+from litellm.types.llms.anthropic import AnthropicMessagesTextParam
+from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat
+
+
+@dataclass(frozen=True, slots=True)
+class InferenceRequestView:
+    messages: Sequence[Mapping[str, object]]
+    system: object | None
+
+
+@dataclass(frozen=True, slots=True)
+class InferenceRequestViewError:
+    error_message: str
+    status_code: int
+
+
+@dataclass(frozen=True, slots=True)
+class CountContextEdits:
+    spec: JsonValue
+    model: str
+    tools: tuple[Mapping[str, object], ...] | None
+    drop_invalid_edits: bool = False
+
+
+_CONTEXT_MANAGEMENT_SPEC: Final[TypeAdapter[Mapping[str, object] | tuple[Mapping[str, object], ...]]] = TypeAdapter(
+    Mapping[str, object] | tuple[Mapping[str, object], ...]
+)
+_SYSTEM_BLOCKS: Final = TypeAdapter(tuple[AnthropicMessagesTextParam, ...])
+
+
+def count_context_edits(
+    context_management: CountContextManagement | None,
+    tools: Sequence[Mapping[str, object]] | None,
+    litellm_params: Mapping[str, object],
+    fallback_model: str,
+) -> CountContextEdits:
+    model: Final = litellm_params.get("model")
+    return CountContextEdits(
+        spec=None if context_management is None else context_management.spec,
+        model=model if isinstance(model, str) else fallback_model,
+        tools=None if tools is None else tuple(tools),
+        drop_invalid_edits=context_management is not None and context_management.drop_invalid_edits,
+    )
+
+
+def _typed_spec(spec: JsonValue) -> Mapping[str, object] | tuple[Mapping[str, object], ...] | None:
+    try:
+        return _CONTEXT_MANAGEMENT_SPEC.validate_python(spec)
+    except ValidationError:
+        return None
+
+
+def _typed_system(system: object | None) -> str | tuple[AnthropicMessagesTextParam, ...] | None | ValidationError:
+    if system is None or isinstance(system, str):
+        return system
+    try:
+        return _SYSTEM_BLOCKS.validate_python(system)
+    except ValidationError as error:
+        return error
+
+
+def _view_without_edits(
+    messages: Sequence[Mapping[str, object]],
+    system: object | None,
+    typed_system: str | Sequence[Mapping[str, object]] | None,
+) -> InferenceRequestView:
+    from litellm.llms.anthropic.messages.context_management.editors.compact import (
+        apply_client_compaction_block_history,
+    )
+
+    history: Final = apply_client_compaction_block_history(
+        messages=[dict(message) for message in messages],
+        system=typed_system
+        if typed_system is None or isinstance(typed_system, str)
+        else [dict(block) for block in typed_system],
+    )
+    if history is None:
+        return InferenceRequestView(messages=messages, system=system)
+    return InferenceRequestView(messages=history.messages, system=history.system)
+
+
+async def _anthropic_inference_view(
+    messages: Sequence[Mapping[str, object]], system: object | None, edits: CountContextEdits
+) -> InferenceRequestView | InferenceRequestViewError:
+    from litellm.llms.anthropic.messages.adapters.handler import prepare_context_managed_request
+    from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
+    from litellm.llms.anthropic.messages.native.utils import normalize_native_messages
+
+    typed_system: Final = _typed_system(system)
+    if isinstance(typed_system, ValidationError):
+        return InferenceRequestView(messages=messages, system=system)
+    normalized: Final = normalize_native_messages(messages)
+    spec: Final = _typed_spec(edits.spec)
+    try:
+        prepared: Final = await prepare_context_managed_request(
+            model=edits.model,
+            messages=normalized,
+            tools=None if edits.tools is None else [dict(tool) for tool in edits.tools],
+            system=[dict(block) for block in typed_system] if isinstance(typed_system, tuple) else typed_system,
+            context_management_spec=(
+                None if spec is None else dict(spec) if isinstance(spec, Mapping) else [dict(edit) for edit in spec]
+            ),
+            litellm_metadata=None,
+            additional_drop_params=None,
+            llm_router=None,
+            count_only=True,
+            drop_invalid_edits=edits.drop_invalid_edits,
+        )
+    except AnthropicContextManagementError as error:
+        if error.status_code >= 500:
+            return _view_without_edits(normalized, system, typed_system)
+        return InferenceRequestViewError(error_message=error.message, status_code=error.status_code)
+    if prepared is None:
+        return InferenceRequestView(messages=normalized, system=system)
+    return InferenceRequestView(messages=prepared.messages, system=prepared.system)
+
+
+async def inference_request_view(
+    messages: Sequence[Mapping[str, object]],
+    system: object | None,
+    messages_format: TokenCountMessagesFormat,
+    edits: CountContextEdits,
+) -> InferenceRequestView | InferenceRequestViewError:
+    """Apply the history rewrites and context edits /v1/messages runs before it converts a body for another provider."""
+    match messages_format:
+        case "openai":
+            return InferenceRequestView(messages=messages, system=system)
+        case "anthropic":
+            return await _anthropic_inference_view(messages, system, edits)
+        case _:
+            assert_never(messages_format)

litellm/llms/anthropic/messages/native/handler.py

@@ -8,6 +8,7 @@
 import asyncio
 import contextvars
 from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from functools import partial
 from typing import Final, cast
 
@@ -15,6 +16,7 @@ from pydantic import TypeAdapter
 
 import litellm
 from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
+from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
 from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.exception_mapping_utils import exception_type
@@ -44,7 +46,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
 )
 from litellm.types.router import GenericLiteLLMParams
-from litellm.types.utils import CallTypes, LlmProviders
+from litellm.types.utils import CallTypes, LlmProviders, LlmProvidersSet
 from litellm.utils import ProviderConfigManager, client
 
 from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler, request_safeguards_evaluator
@@ -94,6 +96,68 @@ def _responses_mode_is_lost_by_prefix_strip(
     ) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info)
 
 
+@dataclass(frozen=True, slots=True)
+class _BridgeTarget:
+    model: str
+    api_key: str | None
+    api_base: str | None
+
+
+def _resolve_requested_provider(
+    model: str, custom_llm_provider: str | None, litellm_params: GenericLiteLLMParams
+) -> tuple[str, str, str | None, str | None]:
+    resolved: Final = litellm.get_llm_provider(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        api_base=litellm_params.api_base,
+        api_key=litellm_params.api_key,
+        litellm_params=litellm_params,
+    )
+    requested_provider: Final = model.partition("/")[0]
+    if (
+        custom_llm_provider in (None, requested_provider)
+        or requested_provider not in LlmProvidersSet
+        or requested_provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
+    ):
+        return resolved
+    own: Final = _resolve_by_model_prefix(model, litellm_params)
+    return own if own is not None and own[1] == custom_llm_provider else resolved
+
+
+def _resolve_by_model_prefix(
+    model: str, litellm_params: GenericLiteLLMParams
+) -> tuple[str, str, str | None, str | None] | None:
+    try:
+        return litellm.get_llm_provider(
+            model=model,
+            api_base=litellm_params.api_base,
+            api_key=litellm_params.api_key,
+            litellm_params=litellm_params.model_copy(update={"custom_llm_provider": None}),
+        )
+    except litellm.BadRequestError:
+        return None
+
+
+def _sets_own_entra_credential(litellm_params: GenericLiteLLMParams) -> bool:
+    return bool(
+        litellm_params.azure_ad_token
+        or (litellm_params.model_extra or {}).get("azure_ad_token_provider") is not None
+        or (litellm_params.tenant_id and litellm_params.client_id and litellm_params.client_secret)
+        or (litellm_params.azure_username and litellm_params.azure_password and litellm_params.client_id)
+    )
+
+
+def _bridge_target(requested: _BridgeTarget, resolved: _BridgeTarget, custom_llm_provider: str) -> _BridgeTarget:
+    requested_provider: Final = requested.model.partition("/")[0]
+    if requested_provider == custom_llm_provider or requested_provider not in LlmProvidersSet:
+        return requested
+    return _BridgeTarget(
+        model=f"{custom_llm_provider}/{resolved.model}",
+        api_key=requested.api_key if requested.api_base else requested.api_key or resolved.api_key,
+        api_base=resolved.api_base,
+    )
+
+
 def _should_route_to_responses_api(
     custom_llm_provider: str | None,
     requested_model: str | None = None,
@@ -624,13 +688,7 @@ def anthropic_messages_handler(
         custom_llm_provider,
         dynamic_api_key,
         dynamic_api_base,
-    ) = litellm.get_llm_provider(
-        model=model,
-        custom_llm_provider=custom_llm_provider,
-        api_base=litellm_params.api_base,
-        api_key=litellm_params.api_key,
-        litellm_params=litellm_params,
-    )
+    ) = _resolve_requested_provider(model, custom_llm_provider, litellm_params)
 
     # Store agentic loop params in logging object for agentic hooks
     # This provides original request context needed for follow-up calls
@@ -656,6 +714,16 @@ def anthropic_messages_handler(
             mock_response=litellm_params.mock_response,
         )
 
+    bridge: Final = _bridge_target(
+        _BridgeTarget(model=original_model, api_key=api_key, api_base=api_base),
+        _BridgeTarget(
+            model=model,
+            api_key=None if _sets_own_entra_credential(litellm_params) else dynamic_api_key,
+            api_base=dynamic_api_base,
+        ),
+        custom_llm_provider,
+    )
+
     # Expand litellm_proxy MCP references through the MCP gateway before dispatch, so every
     # downstream path (native passthrough and both bridges) gets real tools rather than a
     # reference the provider cannot resolve. Popped from kwargs so it never reaches the provider.
@@ -682,8 +750,8 @@ def anthropic_messages_handler(
             top_k=top_k,
             top_p=top_p,
             container=container,
-            api_key=api_key,
-            api_base=api_base,
+            api_key=bridge.api_key,
+            api_base=bridge.api_base,
             client=client,
             custom_llm_provider=custom_llm_provider,
             **kwargs,
@@ -700,7 +768,7 @@ def anthropic_messages_handler(
             return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
                 max_tokens=max_tokens,
                 messages=messages,
-                model=original_model,
+                model=bridge.model,
                 metadata=metadata,
                 stop_sequences=stop_sequences,
                 stream=stream,
@@ -716,8 +784,8 @@ def anthropic_messages_handler(
                 top_k=top_k,
                 top_p=top_p,
                 _is_async=is_async,
-                api_key=api_key,
-                api_base=api_base,
+                api_key=bridge.api_key,
+                api_base=bridge.api_base,
                 client=client,
                 custom_llm_provider=custom_llm_provider,
                 **kwargs,
@@ -730,7 +798,7 @@ def anthropic_messages_handler(
         return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(
             max_tokens=max_tokens,
             messages=messages,
-            model=original_model,
+            model=bridge.model,
             metadata=metadata,
             stop_sequences=stop_sequences,
             stream=stream,
@@ -744,8 +812,8 @@ def anthropic_messages_handler(
             top_k=top_k,
             top_p=top_p,
             _is_async=is_async,
-            api_key=api_key,
-            api_base=api_base,
+            api_key=bridge.api_key,
+            api_base=bridge.api_base,
             client=client,
             custom_llm_provider=custom_llm_provider,
             **kwargs,

litellm/llms/anthropic/messages/responses_adapters/handler.py

@@ -21,9 +21,10 @@ from litellm.litellm_core_utils.asyncify import (
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     cache_control_as_prompt_cache_breakpoint,
-    prompt_cache_breakpoint_for_wire,
+    prompt_cache_breakpoint_with_readable_mode,
     with_prompt_cache_breakpoint,
 )
+from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers
 from litellm.responses.utils import auto_reasoning_summary
 from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
 from litellm.types.llms.anthropic import (
@@ -207,7 +208,7 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str
 def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]:
     if "prompt_cache_breakpoint" not in block:
         return cache_control_as_prompt_cache_breakpoint(block)
-    wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True)
+    wire_marker: Final = prompt_cache_breakpoint_with_readable_mode(block["prompt_cache_breakpoint"], drop_params=True)
     return cache_control_as_prompt_cache_breakpoint(
         with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker)
     )
@@ -272,6 +273,14 @@ def _build_responses_kwargs(
     """
     forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
     reject_malformed_tool_result_text(messages, model, forwarded_kwargs)
+    target_provider: Final = forwarded_kwargs.get("custom_llm_provider")
+    request_tools: Final = tools_without_mcp_servers(
+        forwarded_kwargs.get("mcp_servers"),
+        tools or (),
+        model=model,
+        route=target_provider if isinstance(target_provider, str) else "this route",
+        drop_params=drops_mcp_servers(forwarded_kwargs),
+    )
     block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs)
     request_messages: Final[AnthropicRequestMessages] = (
         [
@@ -304,10 +313,10 @@ def _build_responses_kwargs(
         request_data["temperature"] = temperature
     if thinking:
         request_data["thinking"] = thinking
-    if tool_choice:
+    if tool_choice and (request_tools or not tools):
         request_data["tool_choice"] = tool_choice
-    if tools:
-        request_data["tools"] = tools
+    if request_tools:
+        request_data["tools"] = list(request_tools)
     if top_p is not None:
         request_data["top_p"] = top_p
     if output_format:
@@ -347,7 +356,7 @@ def _build_responses_kwargs(
         responses_kwargs["stream"] = True
 
     # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
-    excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages"}
+    excluded: Final = CONVERTED_STREAM_KEYS | {"anthropic_messages", "mcp_servers"}
     for key, value in forwarded_kwargs.items():
         if key == "litellm_logging_obj" and value is not None:
             from litellm.litellm_core_utils.litellm_logging import (

litellm/llms/azure/chat/gpt_reasoning_transformation.py

@@ -1,6 +1,6 @@
 """Support for Azure OpenAI gpt-5 model family."""
 
-from collections.abc import Collection
+from collections.abc import Collection, Mapping
 from typing import ClassVar, Final
 
 import litellm
@@ -105,7 +105,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
         model: str,
         messages: list[AllMessageValues],
         optional_params: dict,
-        litellm_params: dict,
+        litellm_params: Mapping[str, object],
         headers: dict,
     ) -> dict:
         model = model.replace(self.GPT5_SERIES_ROUTE, "")

litellm/llms/azure/chat/gpt_transformation.py

@@ -5,12 +5,16 @@ from typing import TYPE_CHECKING, Any, Final
 from httpx._models import Headers, Response
 
 import litellm
+from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_cache_breakpoint_support
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    chat_completion_prompt_cache_breakpoints_for_wire,
     drop_tool_reference_parts_from_tool_messages,
     flatten_combinators_and_drop_non_python_regex_patterns,
     hoist_images_from_tool_messages,
     system_messages_first,
     tool_with_sanitized_parameters,
+    without_prompt_cache_breakpoints,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_to_azure_openai_messages,
@@ -290,11 +294,16 @@ class AzureOpenAIConfig(BaseConfig):
         model: str,
         messages: list[AllMessageValues],
         optional_params: dict,
-        litellm_params: dict,
+        litellm_params: Mapping[str, object],
         headers: dict,
     ) -> dict:
         ordered_messages: Final = system_messages_first(messages) if litellm.openai_system_messages_first else messages
-        stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(ordered_messages)
+        wire_messages: Final = (
+            self._with_wire_safe_prompt_cache_breakpoints(model, ordered_messages, litellm_params)
+            if drops_unsupported_params(litellm_params.get("drop_params"))
+            else ordered_messages
+        )
+        stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(wire_messages)
         azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages))
         request_params: Final = MappingProxyType(
             {
@@ -310,6 +319,14 @@ class AzureOpenAIConfig(BaseConfig):
             **sanitized_tools_update(optional_params),
         }
 
+    @staticmethod
+    def _with_wire_safe_prompt_cache_breakpoints(
+        model: str, messages: list[AllMessageValues], litellm_params: Mapping[str, object]
+    ) -> list[AllMessageValues]:
+        if deployment_prompt_cache_breakpoint_support(model, "azure", litellm_params) is False:
+            return [without_prompt_cache_breakpoints(message) for message in messages]
+        return [chat_completion_prompt_cache_breakpoints_for_wire(message, drop_params=True) for message in messages]
+
     def transform_response(
         self,
         model: str,

litellm/llms/azure/responses/transformation.py

@@ -1,6 +1,7 @@
 import re
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from copy import deepcopy
+from functools import partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal
 from urllib.parse import urlsplit
@@ -11,6 +12,7 @@ from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
 from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
+from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cache_breakpoint_within
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
 from litellm.llms.azure.common_utils import BaseAzureLLM
@@ -29,6 +31,7 @@ else:
     LiteLLMLoggingObj = Any
 
 _DECODED_JSON: Final = TypeAdapter(object)
+_AZURE_RESPONSES_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(AzureResponsesPromptCacheBreakpoint)
 
 
 _DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
@@ -81,6 +84,13 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def _is_gpt_5_model(model: str) -> bool:
         return AzureOpenAIGPT5Config.is_model_gpt_5_model(model)
 
+    def _prompt_cache_breakpoint_to_wire(
+        self, litellm_params: GenericLiteLLMParams, reads_breakpoints: bool, known_support: bool | None
+    ) -> Callable[[object], object]:
+        if reads_breakpoints or known_support is True:
+            return partial(prompt_cache_breakpoint_within, _AZURE_RESPONSES_PROMPT_CACHE_BREAKPOINT)
+        return super()._prompt_cache_breakpoint_to_wire(litellm_params, reads_breakpoints, known_support)
+
     @staticmethod
     def _supports_reasoning_effort_none(model: str) -> bool:
         return AzureOpenAIGPT5Config.supports_reasoning_effort_level(model, "none")

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -12,6 +12,7 @@ from pydantic import JsonValue
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.llms.anthropic.common_utils import AnthropicError
@@ -39,6 +40,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
         tools: list[dict[str, JsonValue]] | None = None,
         system: object = None,
         optional_params: Mapping[str, JsonValue] | None = None,
+        context_management: JsonValue = None,
     ) -> dict[str, object]:
         """
         Handle a CountTokens request using httpx with Azure authentication.
@@ -70,6 +72,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
                 tools=tools,
                 system=system,
                 optional_params=optional_params,
+                context_management=context_management,
                 drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")),
             )
 
@@ -80,11 +83,14 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
 
             verbose_logger.debug("Making request to: %s", endpoint_url)
 
+            if context_management is not None:
+                await aload_beta_headers_config()
             # Get required headers with Azure authentication
             headers: Final = self.get_required_headers(
                 api_key=api_key,
                 litellm_params=litellm_params,
                 messages=messages,
+                context_management=context_management,
             )
             if not {"x-api-key", "Authorization"} & headers.keys():
                 raise AnthropicError(

litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py

@@ -135,6 +135,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
                 tools=tools,
                 system=system,
                 optional_params=count_options,
+                context_management=None if context_management is None else context_management.spec,
             )
 
             if result is not None:

litellm/llms/azure_ai/anthropic/count_tokens/transformation.py

@@ -8,9 +8,16 @@ from collections.abc import Sequence
 from typing import Final
 from urllib.parse import urlsplit
 
+from pydantic import JsonValue
+
+from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
 from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
 from litellm.litellm_core_utils.url_utils import join_api_base
-from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
+from litellm.llms.anthropic.common_utils import (
+    context_management_betas,
+    count_tokens_feature_betas,
+    merge_anthropic_beta_headers,
+)
 from litellm.llms.anthropic.count_tokens.transformation import (
     AnthropicCountTokensConfig,
 )
@@ -31,6 +38,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
         api_key: str | None,
         litellm_params: dict[str, object] | None = None,
         messages: Sequence[object] = (),
+        context_management: JsonValue = None,
     ) -> dict[str, str]:
         """
         Get the required headers for the Azure AI Anthropic CountTokens API.
@@ -42,6 +50,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
             api_key: The Azure AI API key
             litellm_params: Optional LiteLLM parameters for additional auth config
             messages: The request messages, read for betas the counted features need
+            context_management: The edit spec the count forwards, read for the betas its edits need
 
         Returns:
             Dictionary of required headers with both x-api-key and Azure authentication
@@ -51,7 +60,11 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
             "Content-Type": "application/json",
             "anthropic-version": "2023-06-01",
             "anthropic-beta": merge_anthropic_beta_headers(
-                ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
+                ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
+                (
+                    *count_tokens_feature_betas("azure_ai", {}, messages),
+                    *filter_and_transform_beta_headers(list(context_management_betas(context_management)), "azure_ai"),
+                ),
             ),
             **({"x-api-key": api_key} if api_key else {}),
         }

litellm/llms/base_llm/google_genai/transformation.py

@@ -164,7 +164,7 @@ class BaseGoogleGenAIGenerateContentConfig(ABC):
         tools: ToolConfigDict | None,
         generate_content_config_dict: dict,
         system_instruction: object | None = None,
-    ) -> dict:
+    ) -> dict[str, object]:
         """
         Transform the request parameters for the generate content API.
 

litellm/llms/bedrock/chat/converse_transformation.py

@@ -52,6 +52,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     parts_of,
 )
 from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
+from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
 from litellm.llms.anthropic.chat.transformation import (
     DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
     DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
@@ -63,6 +64,7 @@ from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
 from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images
 from litellm.llms.bedrock.common_utils import (
+    bedrock_chat_opts_into_native_structured_output,
     bedrock_chat_uses_native_structured_output,
     bedrock_cleaned_output_schema,
     bedrock_model_is_unknown,
@@ -933,8 +935,9 @@ class AmazonConverseConfig(BaseConfig):
         """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request.
 
         ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and
-        ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) only when the
-        request asks for thinking, which the json tool path would drop.
+        ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when the
+        request asks for thinking, which the json tool path would drop. Without thinking, a model that cannot
+        force the json tool gets it unless that key is false.
         Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
         """
         return bedrock_chat_uses_native_structured_output(
@@ -1068,20 +1071,23 @@ class AmazonConverseConfig(BaseConfig):
         drop_params: bool,
         model_info: Mapping[str, object] | None = None,
     ) -> dict:
-        is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params)
+        request_params: Final = tool_params_without_hosted_mcp(
+            non_default_params, model=model, route="bedrock", drop_params=drop_params
+        )
+        is_thinking_enabled: Final = self.is_thinking_enabled(request_params)
         base_model: Final = BedrockModelInfo.get_base_model(model)
         drop_thinking_param: Final = self._is_deepseek_model(model=model, base_model=base_model)
         drop_reasoning_effort_param: Final = self._model_rejects_reasoning_effort_param(
             model=model, base_model=base_model
         )
 
-        for param, value in non_default_params.items():
+        for param, value in request_params.items():
             if param == "response_format" and isinstance(value, dict):
                 optional_params = self._translate_response_format_param(
                     value=value,
                     model=model,
                     optional_params=optional_params,
-                    non_default_params=non_default_params,
+                    non_default_params=request_params,
                     is_thinking_enabled=is_thinking_enabled,
                     model_info=model_info,
                 )
@@ -1135,7 +1141,7 @@ class AmazonConverseConfig(BaseConfig):
                         ),
                     ),
                     model=model,
-                    non_default_params=non_default_params,
+                    non_default_params=request_params,
                     optional_params=optional_params,
                 )
             if param == "tool_choice":
@@ -1160,7 +1166,7 @@ class AmazonConverseConfig(BaseConfig):
                     and value.get("type") == "adaptive"
                     and not AnthropicConfig.is_adaptive_thinking_model(model, "bedrock")
                 ):
-                    max_tokens = non_default_params.get("max_completion_tokens") or non_default_params.get("max_tokens")
+                    max_tokens = request_params.get("max_completion_tokens") or request_params.get("max_tokens")
                     legacy_thinking = AnthropicConfig.map_reasoning_effort(
                         reasoning_effort="medium",
                         model=model,
@@ -1230,7 +1236,7 @@ class AmazonConverseConfig(BaseConfig):
         # Nova 2 handles token budgeting differently through reasoningConfig
         if "gpt-oss" not in model and not self._is_nova_2_model(model):
             self.update_optional_params_with_thinking_tokens(
-                non_default_params=non_default_params, optional_params=optional_params
+                non_default_params=request_params, optional_params=optional_params
             )
 
         final_is_thinking_enabled: Final = self.is_thinking_enabled(optional_params)
@@ -1312,19 +1318,15 @@ class AmazonConverseConfig(BaseConfig):
         if "type" in value and value["type"] == "text":
             return optional_params
 
+        thinking_requested: Final = self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params))
         native_schema: Final = (
-            self._native_output_schema(
-                model,
-                json_schema,
-                self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)),
-                model_info,
-            )
+            self._native_output_schema(model, json_schema, thinking_requested, model_info)
             if json_schema is not None
             else None
         )
         if native_schema is not None:
             # Use Bedrock's native structured outputs API (outputConfig.textFormat)
-            # No synthetic tool injection, no fake_stream needed.
+            # No synthetic tool injection.
             # Requires an explicit schema — json_object with no schema falls through
             # to the tool-call path below.
             output_config: Final = self._create_output_config_for_response_format(
@@ -1350,8 +1352,6 @@ class AmazonConverseConfig(BaseConfig):
                 optional_params["tool_choice"] = ToolChoiceValuesBlock(
                     tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)
                 )
-            if non_default_params.get("stream", False) is True:
-                optional_params["fake_stream"] = True
         # else: response_format=json_object with no schema.
         # Don't inject the synthetic json_tool_call tool here. When no
         # schema is given, _create_json_tool_call_for_response_format
@@ -1359,6 +1359,13 @@ class AmazonConverseConfig(BaseConfig):
         # returns {} instead of the requested JSON. The model already
         # returns JSON when the prompt asks for it.
 
+        # A request that goes native only because the model can't force the json tool keeps the json tool's faked
+        # stream, so it still needs only bedrock:InvokeModel.
+        streams_natively: Final = native_schema is not None and bedrock_chat_opts_into_native_structured_output(
+            model, thinking_requested, self.custom_llm_provider or "bedrock_converse", model_info
+        )
+        if json_schema is not None and not streams_natively and non_default_params.get("stream", False) is True:
+            optional_params["fake_stream"] = True
         optional_params["json_mode"] = True
         return optional_params
 

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py

@@ -1,11 +1,13 @@
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
 from pydantic import TypeAdapter
 
+import litellm
 from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body
 from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
+from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
 from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_to_anthropic_image_obj,
@@ -25,7 +27,9 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
 )
 from litellm.llms.bedrock.common_utils import (
     BEDROCK_INVOKE_TRACE_FIELD,
+    BedrockModelInfo,
     apply_bedrock_invoke_structured_output,
+    bedrock_capability_model,
     bedrock_chat_uses_native_structured_output,
     bedrock_cleaned_output_schema,
     bedrock_supports_tool_search,
@@ -35,6 +39,7 @@ from litellm.llms.bedrock.common_utils import (
     normalize_custom_field_on_tools,
     normalize_tool_input_schema_types_for_bedrock_invoke,
     requests_opt_in_native_output,
+    strip_bedrock_routing_prefix,
     strip_unsupported_bedrock_invoke_output_config_keys,
     tools_without_eager_input_streaming,
 )
@@ -59,6 +64,19 @@ _NON_DEFAULT_PARAMS: Final = TypeAdapter(Mapping[str, object])
 _LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
 
 
+def _anthropic_config_max_tokens() -> int | None:
+    return litellm.AnthropicConfig.max_tokens
+
+
+def _mapped_max_tokens(model: str) -> int | None:
+    row: Final = object_mapping(litellm.model_cost.get(model)) or {}  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # cost map rows are untyped JSON
+    match row.get("max_output_tokens"):
+        case int() as max_tokens:
+            return max_tokens
+        case _:
+            return None
+
+
 class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
     """
     Reference:
@@ -79,9 +97,18 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
 
     anthropic_version: str = "bedrock-2023-05-31"
 
-    def __init__(self, model_info: Mapping[str, object] | None = None) -> None:
+    def __init__(
+        self,
+        model_info: Mapping[str, object] | None = None,
+        read_configured_max_tokens: Callable[[], int | None] = _anthropic_config_max_tokens,
+    ) -> None:
         super().__init__()
         self._deployment_model_info: Final = model_info
+        self._read_configured_max_tokens: Final = read_configured_max_tokens
+
+    @classmethod
+    def get_config(cls, *, model: str | None = None) -> dict[str, object]:  # mutable-ok: overrides BaseConfig's dict
+        return {key: value for key, value in super().get_config(model=model).items() if key != "max_tokens"}
 
     @property
     def custom_llm_provider(self) -> str | None:
@@ -102,10 +129,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         json_schema: Mapping[str, object] | None,
         request_params: Mapping[str, object],
     ) -> bool:
-        return bedrock_chat_uses_native_structured_output(
-            model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info
+        return self.chat_uses_native_structured_output(
+            model, self.is_thinking_requested(request_params), self._deployment_model_info
         ) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
 
+    def chat_uses_native_structured_output(
+        self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None
+    ) -> bool:
+        return bedrock_chat_uses_native_structured_output(model, thinking_requested, model_info=model_info)
+
     def uses_opt_in_native_output(
         self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
     ) -> bool:
@@ -241,6 +273,36 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
             headers=headers,
         )
 
+    def _default_max_tokens_models(self, model: str, litellm_params: Mapping[str, object]) -> tuple[str, ...]:
+        return (
+            BedrockModelInfo.get_base_model(
+                bedrock_capability_model(strip_bedrock_routing_prefix(model), litellm_params.get("base_model"))
+            ),
+        )
+
+    def _default_max_tokens(self, model: str, litellm_params: Mapping[str, object]) -> int:
+        configured_max_tokens: Final = self._read_configured_max_tokens()
+        match deployment_model_info(litellm_params).get("max_output_tokens"):
+            case int() as deployment_max_tokens:
+                return (
+                    deployment_max_tokens
+                    if configured_max_tokens is None
+                    else min(deployment_max_tokens, configured_max_tokens)
+                )
+            case _ if configured_max_tokens is not None:
+                return configured_max_tokens
+            case _:
+                return next(
+                    (
+                        max_tokens
+                        for max_tokens in map(
+                            _mapped_max_tokens, self._default_max_tokens_models(model, litellm_params)
+                        )
+                        if max_tokens is not None
+                    ),
+                    DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS,
+                )
+
     def _build_bedrock_anthropic_request_base(
         self,
         model: str,
@@ -257,23 +319,35 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
                 model=model,
                 output_config=filtered_params["output_config"],
             )
-        filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params)
+        normalized_params: Final = self._normalize_bedrock_tool_search_tools(filtered_params)
+        default_max_tokens: Final = (
+            None
+            if "max_tokens" in normalized_params
+            else self._default_max_tokens(model, _LITELLM_PARAMS.validate_python(litellm_params))
+        )
+        request_params: Final = (
+            normalized_params if default_max_tokens is None else {**normalized_params, "max_tokens": default_max_tokens}
+        )
 
         anthropic_request: Final = AnthropicConfig.transform_request(
             self,
             model=model,
             messages=messages,
-            optional_params=filtered_params,
+            optional_params=request_params,
             litellm_params=litellm_params,
             headers=headers,
         )
 
         anthropic_request.pop("model", None)
         anthropic_request.pop("stream", None)
+        model_info: Final = deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params))
         apply_bedrock_invoke_structured_output(
             model=model,
             request_body=anthropic_request,
-            model_info=deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params)),
+            model_info=model_info,
+            chat_uses_native_output=lambda: self.chat_uses_native_structured_output(
+                model, self.is_thinking_requested(_NON_DEFAULT_PARAMS.validate_python(filtered_params)), model_info
+            ),
         )
         strip_unsupported_bedrock_invoke_output_config_keys(
             model=model,
@@ -377,7 +451,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
                     "data": image_chunk["data"],
                 }
 
-    def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict:
+    def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict[str, object]:
         """
         Convert tool search entries to the format supported by the Bedrock Invoke API.
         """

litellm/llms/bedrock/chat/mantle/transformation.py

@@ -15,7 +15,7 @@ from litellm.llms.anthropic.chat.transformation import JsonModeRequest
 from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import (
     AmazonAnthropicClaudeConfig,
 )
-from litellm.llms.bedrock.common_utils import build_mantle_messages_url
+from litellm.llms.bedrock.common_utils import bedrock_chat_opts_into_native_structured_output, build_mantle_messages_url
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import ModelResponse
 
@@ -54,6 +54,14 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
             region=region,
         )
 
+    def chat_uses_native_structured_output(
+        self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None
+    ) -> bool:
+        return bedrock_chat_opts_into_native_structured_output(model, thinking_requested, model_info=model_info)
+
+    def _default_max_tokens_models(self, model: str, litellm_params: Mapping[str, object]) -> tuple[str, ...]:
+        return (f"bedrock_mantle/{model}", *super()._default_max_tokens_models(model, litellm_params))
+
     def validate_environment(
         self,
         headers: dict,

litellm/llms/bedrock/claude_platform/transformation.py

@@ -30,6 +30,9 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig):
     def should_strip_billing_metadata(self) -> bool:
         return True
 
+    def runs_hosted_mcp(self) -> bool:
+        return True
+
     def validate_environment(
         self,
         headers: dict,

litellm/llms/bedrock/common_utils.py

@@ -10,7 +10,7 @@ import json
 import os
 import re
 import warnings
-from collections.abc import Iterator, Mapping, Sequence
+from collections.abc import Callable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
@@ -36,6 +36,7 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.litellm_core_utils.optional_imports import ensure_optional_import
 from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
 from litellm.litellm_core_utils.url_utils import ensure_api_route
+from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
 )
@@ -43,6 +44,7 @@ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned
 from litellm.secret_managers.main import get_secret, get_secret_str
+from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
 from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams
 
 if TYPE_CHECKING:
@@ -50,6 +52,12 @@ if TYPE_CHECKING:
     from litellm.types.llms.openai import AllMessageValues
 
 BEDROCK_INVOKE_TRACE_FIELD: Final = "amazon-bedrock-trace"
+BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
+        "clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
+    }
+)
 
 
 _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
@@ -308,7 +316,33 @@ def bedrock_supports_native_structured_output(
     ) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
 
 
-def bedrock_chat_uses_native_structured_output(
+def _bedrock_map_native_output_flag(model: str) -> object:
+    routed: Final = strip_bedrock_routing_prefix(model)
+    flags: Final = (
+        (entry or {}).get("supports_bedrock_native_structured_output")
+        for entry in (
+            _model_cost_entry(f"bedrock/{routed}"),
+            _model_cost_entry(routed),
+            _model_cost_entry(split_bedrock_region_path(model)[1]),
+        )
+    )
+    return next((flag for flag in flags if isinstance(flag, bool)), None)
+
+
+def _bedrock_native_output_opted_out(model: str, model_info: Mapping[str, object] | None) -> bool:
+    deployment_flag: Final = (model_info or {}).get("supports_bedrock_native_structured_output")
+    if isinstance(deployment_flag, bool):
+        return not deployment_flag
+    return _bedrock_map_native_output_flag(model) is False
+
+
+def _bedrock_native_output_replaces_unforceable_tool(model: str, model_info: Mapping[str, object] | None) -> bool:
+    return AnthropicModelInfo.forced_tool_use_unsupported(model) and not _bedrock_native_output_opted_out(
+        model, model_info
+    )
+
+
+def bedrock_chat_opts_into_native_structured_output(
     model: str,
     thinking_requested: bool,
     custom_llm_provider: str = "bedrock",
@@ -319,7 +353,18 @@ def bedrock_chat_uses_native_structured_output(
     )
 
 
-def bedrock_native_output_is_opt_in(
+def bedrock_chat_uses_native_structured_output(
+    model: str,
+    thinking_requested: bool,
+    custom_llm_provider: str = "bedrock",
+    model_info: Mapping[str, object] | None = None,
+) -> bool:
+    return bedrock_chat_opts_into_native_structured_output(
+        model, thinking_requested, custom_llm_provider, model_info
+    ) or (not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info))
+
+
+def bedrock_native_output_is_deployment_opt_in(
     model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
 ) -> bool:
     return not _bedrock_model_supports(
@@ -327,6 +372,15 @@ def bedrock_native_output_is_opt_in(
     ) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
 
 
+def bedrock_native_output_is_opt_in(
+    model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
+) -> bool:
+    return not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) and (
+        _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
+        or _bedrock_native_output_replaces_unforceable_tool(model, model_info)
+    )
+
+
 def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
     return {
         **litellm_params,
@@ -417,13 +471,18 @@ def requests_output_config_format(optional_params: Mapping[str, object]) -> bool
             return False
 
 
+def requests_native_output(anthropic_optional_params: Mapping[str, object]) -> bool:
+    return isinstance(anthropic_optional_params.get("output_format"), Mapping) or requests_output_config_format(
+        anthropic_optional_params
+    )
+
+
 def requests_opt_in_native_output(
     model: str, anthropic_optional_params: Mapping[str, object], model_info: Mapping[str, object] | None
 ) -> bool:
-    native_output_requested: Final = isinstance(
-        anthropic_optional_params.get("output_format"), Mapping
-    ) or requests_output_config_format(anthropic_optional_params)
-    return native_output_requested and bedrock_native_output_is_opt_in(model, model_info=model_info)
+    return requests_native_output(anthropic_optional_params) and bedrock_native_output_is_opt_in(
+        model, model_info=model_info
+    )
 
 
 _JSON_OBJECT: Final = TypeAdapter(dict[str, object])
@@ -484,6 +543,7 @@ def apply_bedrock_invoke_structured_output(
     model: str,
     request_body: dict[str, object],  # mutable-ok: edited in place like siblings
     model_info: Mapping[str, object] | None = None,
+    chat_uses_native_output: Callable[[], bool] = lambda: False,
 ) -> None:
     """
     Route Anthropic structured-output params to what the Bedrock model supports.
@@ -495,7 +555,8 @@ def apply_bedrock_invoke_structured_output(
     forwarded as ``output_config.format``, which Bedrock relays to the model for
     enforced structured output. For every other model the schema is inlined into
     the last user message as best-effort text, with a warning because nothing
-    enforces it.
+    enforces it. ``chat_uses_native_output`` is asked only when a schema is
+    present, and a true answer sends it natively after the same cleanup.
     """
     legacy_output_format: Final = request_body.pop("output_format", None)
     output_config_format: Final = pop_bedrock_invoke_output_config_format(request_body)
@@ -504,7 +565,15 @@ def apply_bedrock_invoke_structured_output(
     if schema_format is None:
         return
 
-    native_format: Final = bedrock_native_output_format(model, schema_format, model_info=model_info)
+    native_format: Final = bedrock_native_output_format(
+        model,
+        schema_format,
+        model_info=(
+            {**(model_info or {}), "supports_bedrock_native_structured_output": True}
+            if chat_uses_native_output()
+            else model_info
+        ),
+    )
     if native_format is not None:
         existing_output_config: Final = request_body.get("output_config")
         if isinstance(existing_output_config, dict):

litellm/llms/bedrock/count_tokens/bedrock_token_counter.py

@@ -13,9 +13,14 @@ from typing_extensions import assert_never
 import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
+from litellm.llms.anthropic.common_utils import context_management_betas, context_management_with_edits
 from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, anthropic_count_fields
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
-from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
+from litellm.llms.bedrock.common_utils import (
+    BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS,
+    BedrockError,
+    get_bedrock_base_model,
+)
 from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
 from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
 from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model
@@ -74,6 +79,16 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
     )
 
 
+def _mantle_retry_outcome(
+    runtime: CountTokensOutcome, mantle: CountTokensOutcome, carried_edits: bool
+) -> CountTokensOutcome:
+    match runtime, mantle:
+        case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404) if carried_edits:
+            return runtime
+        case _:
+            return mantle
+
+
 def _request_data(
     model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, system: object
 ) -> Mapping[str, object]:
@@ -85,6 +100,33 @@ def _request_data(
     }
 
 
+def _invoke_model_message(message: Mapping[str, object]) -> Mapping[str, object]:
+    content: Final = message.get("content")
+    return (
+        {**message, "content": [{"type": "text", "text": content}]} if isinstance(content, str) and content else message
+    )
+
+
+def _invoke_supports_edit(edit: JsonValue) -> bool:
+    edit_type: Final = edit.get("type") if isinstance(edit, dict) else None
+    return isinstance(edit_type, str) and edit_type in BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS
+
+
+def _invoke_context_management(context_management: CountContextManagement | None) -> Mapping[str, JsonValue]:
+    spec: Final = context_management_with_edits(None if context_management is None else context_management.spec)
+    edits: Final = spec.get("edits") if isinstance(spec, dict) else None
+    supported_edits: Final = tuple(filter(_invoke_supports_edit, edits if isinstance(edits, list) else ()))
+    if not isinstance(spec, dict) or not supported_edits:
+        return MappingProxyType({})
+    supported_spec: Final[JsonValue] = {**spec, "edits": list(supported_edits)}
+    return MappingProxyType(
+        {
+            "context_management": supported_spec,
+            "anthropic_beta": list(context_management_betas(supported_spec)),
+        }
+    )
+
+
 class BedrockTokenCounter(BaseTokenCounter):
     """Counts on the configured Bedrock endpoint, with Mantle retry when Runtime rejects a Claude model"""
 
@@ -167,11 +209,19 @@ class BedrockTokenCounter(BaseTokenCounter):
         )
         if isinstance(translated, ChatCountTranslationError):
             return translated.response(request_model, model_to_use, primary_tokenizer)
-        runtime_request: Final = (
-            _request_data(model_to_use, messages, tools, system)
-            if translated is None
-            else _request_data(model_to_use, translated.messages, translated.tools, translated.system)
+        native_context_management: Final = _invoke_context_management(
+            context_management if is_mantle_claude_model(resolved_model) else None
         )
+        counted_messages: Final = messages if translated is None else translated.messages
+        runtime_request: Final = {
+            **_request_data(
+                model_to_use,
+                tuple(map(_invoke_model_message, counted_messages)) if native_context_management else counted_messages,
+                tools if translated is None else translated.tools,
+                system if translated is None else translated.system,
+            ),
+            **native_context_management,
+        }
         mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options}
 
         primary: Final = (
@@ -184,8 +234,12 @@ class BedrockTokenCounter(BaseTokenCounter):
             )
         )
         outcome: Final = (
-            await self._count_with(
-                self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model
+            _mantle_retry_outcome(
+                primary,
+                await self._count_with(
+                    self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model
+                ),
+                carried_edits=bool(native_context_management),
             )
             if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model)
             else primary

litellm/llms/bedrock/count_tokens/mantle_handler.py

@@ -8,7 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.url_utils import join_api_base
-from litellm.llms.anthropic.common_utils import tools_without_custom_field
+from litellm.llms.anthropic.common_utils import context_management_betas, tools_without_custom_field
 from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
 from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url
@@ -22,6 +22,7 @@ class MantleCountTokensRequest(TypedDict):
     messages: ReadOnly[list[dict[str, JsonValue]]]
     system: ReadOnly[NotRequired[JsonValue]]
     tools: ReadOnly[NotRequired[list[dict[str, JsonValue]]]]
+    context_management: ReadOnly[NotRequired[JsonValue]]
 
 
 _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest)
@@ -74,13 +75,19 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
                 optional_params=_COUNT_OPTIONS.validate_python(
                     {name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES}
                 ),
+                context_management=request.get("context_management"),
             )
+            betas: Final = context_management_betas(request.get("context_management"))
             verbose_logger.debug("Making bedrock-mantle count_tokens request to: %s", endpoint_url)
             api_key: Final = litellm_params.get("api_key")
             signed_headers, signed_body = await run_aws_signing(
                 self._sign_request,
                 service_name="bedrock",
-                headers={"Content-Type": "application/json", "anthropic-version": MANTLE_ANTHROPIC_VERSION},
+                headers={
+                    "Content-Type": "application/json",
+                    "anthropic-version": MANTLE_ANTHROPIC_VERSION,
+                    **({"anthropic-beta": ",".join(betas)} if betas else {}),
+                },
                 optional_params=litellm_params,
                 request_data=body,
                 api_base=endpoint_url,

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py

@@ -37,8 +37,10 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
     bedrock_invoke_guardrail_headers,
 )
 from litellm.llms.bedrock.common_utils import (
+    BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS,
     BedrockError,
     apply_bedrock_invoke_structured_output,
+    bedrock_native_output_is_deployment_opt_in,
     bedrock_supports_tool_search,
     ensure_bedrock_anthropic_messages_tool_names,
     get_anthropic_beta_from_headers,
@@ -46,7 +48,7 @@ from litellm.llms.bedrock.common_utils import (
     normalize_bedrock_opus_output_config_effort,
     normalize_custom_field_on_tools,
     normalize_tool_input_schema_types_for_bedrock_invoke,
-    requests_opt_in_native_output,
+    requests_native_output,
     requests_output_config_format,
     strip_unsupported_bedrock_invoke_output_config_keys,
     supports_bedrock_prompt_cache_ttl,
@@ -130,7 +132,10 @@ class AmazonAnthropicClaudeMessagesConfig(
         litellm_params: GenericLiteLLMParams,
     ) -> GenericLiteLLMParams | None:
         model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info)
-        if not requests_opt_in_native_output(model, anthropic_messages_optional_request_params, model_info):
+        if not (
+            requests_native_output(anthropic_messages_optional_request_params)
+            and bedrock_native_output_is_deployment_opt_in(model, model_info=model_info)
+        ):
             return None
         return litellm_params.model_copy(
             update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}}
@@ -499,12 +504,7 @@ class AmazonAnthropicClaudeMessagesConfig(
     # Bedrock InvokeModel DOES support ``clear_tool_uses_20250919`` under the
     # ``context-management-2025-06-27`` beta. AWS docs:
     # https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-tool-use.md
-    _BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Mapping[str, str] = MappingProxyType(
-        {
-            "compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
-            "clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
-        }
-    )
+    _BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Mapping[str, str] = BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS
 
     @classmethod
     def _filter_context_management_for_bedrock_invoke(

litellm/llms/bedrock_mantle/responses/transformation.py

@@ -15,7 +15,8 @@ role / access key / profile / web identity), signed via the shared
 BaseAWSLLM._sign_request after the request body is finalized.
 """
 
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
+from functools import partial
 from typing import Final, cast  # noqa: TID251  # map_openai_params returns the filtered params as a bare dict
 
 import httpx
@@ -23,6 +24,7 @@ import httpx
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
+from litellm.litellm_core_utils.prompt_templates.common_utils import chat_completion_prompt_cache_breakpoint_for_wire
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
@@ -62,6 +64,10 @@ _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
 _BEDROCK_MANTLE_OPENAI_PATH_SUPPORTED_REASONING_SUMMARIES: Final = frozenset({"auto"})
 
 
+def _prompt_cache_breakpoint_as_sent_or_none(marker: object) -> object:
+    return marker if chat_completion_prompt_cache_breakpoint_for_wire(marker, drop_params=True) == marker else None
+
+
 class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
     def __init__(
         self,
@@ -113,6 +119,13 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
             headers["OpenAI-Project"] = litellm_params.aws_bedrock_project_id
         return headers
 
+    def _prompt_cache_breakpoint_to_wire(
+        self, litellm_params: GenericLiteLLMParams, reads_breakpoints: bool, known_support: bool | None
+    ) -> Callable[[object], object]:
+        if reads_breakpoints or known_support is True:
+            return partial(chat_completion_prompt_cache_breakpoint_for_wire, drop_params=True)
+        return _prompt_cache_breakpoint_as_sent_or_none
+
     def supports_native_file_search(self) -> bool:
         return False
 

litellm/llms/cohere/chat/v2_transformation.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Final
 import httpx
 
 import litellm
+from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
 from litellm.types.llms.cohere import CohereV2ChatResponse
@@ -248,6 +249,9 @@ class CohereV2ChatConfig(OpenAIGPTConfig):
         prompt_tokens: Final = token_usage.get("input_tokens", 0)
         completion_tokens: Final = token_usage.get("output_tokens", 0)
 
+        model_response.choices[0].finish_reason = map_finish_reason(  # rebind-ok: ModelResponse is filled in place
+            cohere_v2_chat_response.get("finish_reason", "COMPLETE")
+        )
         model_response.created = int(time.time())
         model_response.model = model
         usage: Final = Usage(

litellm/llms/custom_httpx/llm_http_handler.py

@@ -38,7 +38,11 @@ import litellm.litellm_core_utils
 import litellm.types
 import litellm.types.utils
 from litellm._logging import redact_string, verbose_logger
-from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, update_headers_with_filtered_beta
+from litellm.anthropic_beta_headers_manager import (
+    aload_beta_headers_config,
+    resolve_beta_provider,
+    update_headers_with_filtered_beta,
+)
 from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
 from litellm.files.types import FileContentStreamingResult
 from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
@@ -455,6 +459,20 @@ def _google_genai_streaming_hidden_params(
     }
 
 
+def _log_generate_content_request(
+    logging_obj: LiteLLMLoggingObj,
+    contents: object,
+    api_base: str,
+    headers: Mapping[str, object],
+    body: Mapping[str, object],
+) -> None:
+    logging_obj.pre_call(
+        input=contents,
+        api_key="",
+        additional_args={"complete_input_dict": body, "api_base": api_base, "headers": headers},
+    )
+
+
 @lru_cache(maxsize=None)
 def _responses_api_optional_request_param_names() -> frozenset[str]:
     return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams).keys())
@@ -2581,14 +2599,16 @@ class BaseLLMHTTPHandler:
             litellm_params.get("additional_drop_params") or (),
             anthropic_messages_optional_request_params,
         )
-        if (
-            request_params.get("safeguards") is not None
-            and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
-        ):
-            await aload_beta_headers_config()
         affinity_api_base: Final = (
             AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
         )
+        takes_another_beta_row: Final = (
+            resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider
+        )
+        provider_filters_betas: Final = anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
+        filters_beta_headers: Final = provider_filters_betas or takes_another_beta_row
+        if request_params.get("safeguards") is not None and filters_beta_headers:
+            await aload_beta_headers_config()
         default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
         (
             headers,
@@ -2607,8 +2627,14 @@ class BaseLLMHTTPHandler:
             api_base=api_base,
         )
 
-        if anthropic_messages_provider_config.should_filter_anthropic_beta_headers():
-            headers = update_headers_with_filtered_beta(headers=headers, provider=custom_llm_provider)
+        if filters_beta_headers and headers.get("anthropic-beta") is not None:
+            await aload_beta_headers_config()
+            headers = update_headers_with_filtered_beta(
+                headers=headers,
+                provider=custom_llm_provider,
+                api_base=affinity_api_base,
+                unlisted="drop" if provider_filters_betas else "forward",
+            )
 
         from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 
@@ -12443,6 +12469,10 @@ class BaseLLMHTTPHandler:
         from litellm.google_genai.streaming_iterator import (
             GoogleGenAIGenerateContentStreamingIterator,
         )
+        from litellm.llms.vertex_ai.gemini.rejected_thought_signature_retry import (
+            post_with_accepted_thought_signatures_sync,
+            with_placeholder_native_thought_signatures,
+        )
 
         if _is_async:
             return self.async_generate_content_handler(
@@ -12494,27 +12524,19 @@ class BaseLLMHTTPHandler:
         if extra_body:
             data.update(extra_body)
 
-        ## LOGGING
-        logging_obj.pre_call(
-            input=contents,
-            api_key="",
-            additional_args={
-                "complete_input_dict": data,
-                "api_base": api_base,
-                "headers": headers,
-            },
-        )
+        log_request: Final = partial(_log_generate_content_request, logging_obj, contents, api_base, headers)
+        log_request(data)
+
+        def post(body: Mapping[str, object]) -> httpx.Response:
+            return sync_httpx_client.post(
+                url=api_base, headers=headers, json=dict(body), timeout=timeout, stream=stream
+            )
 
         try:
+            sent_body, response = post_with_accepted_thought_signatures_sync(
+                data, with_placeholder_native_thought_signatures, None, post, log_request
+            )
             if stream:
-                response = sync_httpx_client.post(
-                    url=api_base,
-                    headers=headers,
-                    json=data,
-                    timeout=timeout,
-                    stream=True,
-                )
-                # Return streaming iterator
                 return GoogleGenAIGenerateContentStreamingIterator(
                     response=response,
                     model=model,
@@ -12522,7 +12544,7 @@ class BaseLLMHTTPHandler:
                     generate_content_provider_config=generate_content_provider_config,
                     litellm_metadata=litellm_metadata or {},
                     custom_llm_provider=custom_llm_provider,
-                    request_body=data,
+                    request_body=dict(sent_body),
                     hidden_params=_google_genai_streaming_hidden_params(
                         api_base=api_base,
                         litellm_params=litellm_params,
@@ -12530,13 +12552,6 @@ class BaseLLMHTTPHandler:
                         response_headers=response.headers,
                     ),
                 )
-            else:
-                response = sync_httpx_client.post(
-                    url=api_base,
-                    headers=headers,
-                    json=data,
-                    timeout=timeout,
-                )
         except Exception as e:
             raise self._handle_error(
                 e=e,
@@ -12574,6 +12589,10 @@ class BaseLLMHTTPHandler:
         from litellm.google_genai.streaming_iterator import (
             AsyncGoogleGenAIGenerateContentStreamingIterator,
         )
+        from litellm.llms.vertex_ai.gemini.rejected_thought_signature_retry import (
+            post_with_accepted_thought_signatures_async,
+            with_placeholder_native_thought_signatures,
+        )
 
         if client is None or not isinstance(client, AsyncHTTPHandler):
             async_httpx_client = get_async_httpx_client(
@@ -12609,27 +12628,19 @@ class BaseLLMHTTPHandler:
         if extra_body:
             data.update(extra_body)
 
-        ## LOGGING
-        logging_obj.pre_call(
-            input=contents,
-            api_key="",
-            additional_args={
-                "complete_input_dict": data,
-                "api_base": api_base,
-                "headers": headers,
-            },
-        )
+        log_request: Final = partial(_log_generate_content_request, logging_obj, contents, api_base, headers)
+        log_request(data)
+
+        async def post(body: Mapping[str, object]) -> httpx.Response:
+            return await async_httpx_client.post(
+                url=api_base, headers=headers, json=dict(body), timeout=timeout, stream=stream
+            )
 
         try:
+            sent_body, response = await post_with_accepted_thought_signatures_async(
+                data, with_placeholder_native_thought_signatures, None, post, log_request
+            )
             if stream:
-                response = await async_httpx_client.post(
-                    url=api_base,
-                    headers=headers,
-                    json=data,
-                    timeout=timeout,
-                    stream=True,
-                )
-                # Return async streaming iterator
                 return AsyncGoogleGenAIGenerateContentStreamingIterator(
                     response=response,
                     model=model,
@@ -12637,7 +12648,7 @@ class BaseLLMHTTPHandler:
                     generate_content_provider_config=generate_content_provider_config,
                     litellm_metadata=litellm_metadata or {},
                     custom_llm_provider=custom_llm_provider,
-                    request_body=data,
+                    request_body=dict(sent_body),
                     hidden_params=_google_genai_streaming_hidden_params(
                         api_base=api_base,
                         litellm_params=litellm_params,
@@ -12645,13 +12656,6 @@ class BaseLLMHTTPHandler:
                         response_headers=response.headers,
                     ),
                 )
-            else:
-                response = await async_httpx_client.post(
-                    url=api_base,
-                    headers=headers,
-                    json=data,
-                    timeout=timeout,
-                )
         except Exception as e:
             raise self._handle_error(
                 e=e,

litellm/llms/databricks/chat/transformation.py

@@ -4,6 +4,7 @@ Translates from OpenAI's `/v1/chat/completions` to Databricks' `/chat/completion
 
 import os
 from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
+from itertools import chain, takewhile
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
 
@@ -12,6 +13,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter
 
 from litellm.anthropic_beta_gated_fields import log_beta_gated_strips, strip_beta_gated_thinking
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
     handle_invalid_parallel_tool_calls,
     should_convert_tool_call_to_json_mode,
@@ -23,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     strip_name_from_message,
     strip_prefill_prefix_field,
 )
+from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.base_utils import (
@@ -65,7 +68,7 @@ from ...anthropic.chat.transformation import (
     tools_without_unsupported_tool_search,
 )
 from ...openai_like.chat.transformation import OpenAILikeChatConfig
-from ..common_utils import DatabricksBase, DatabricksException
+from ..common_utils import DATABRICKS_HOSTED_MODEL_PREFIXES, DatabricksBase, DatabricksException
 
 _RESPONSE_FORMAT_ADAPTER: (  # mutable-ok: mirrors the dict return contract of get_json_schema_from_pydantic_object
     Final[TypeAdapter[dict[str, object] | None]]
@@ -81,15 +84,139 @@ def _is_bare_assistant_message(message_dict: Mapping[str, object]) -> bool:
     )
 
 
+_CONTENT_BLOCKS: Final = TypeAdapter(tuple[object, ...])
+
+
+def _content_blocks(message: object) -> tuple[object, ...]:
+    match message.model_dump(exclude_none=True) if isinstance(message, BaseModel) else message:
+        case {"content": list() as content}:
+            return _CONTENT_BLOCKS.validate_python(content)
+        case _:
+            return ()
+
+
+def _plain_text(block: object) -> str | None:
+    match block:
+        case {"type": "text", "text": str() as text} if block == {"type": "text", "text": text}:
+            return text
+        case _:
+            return None
+
+
+def _plain_texts(message: Mapping[str, object]) -> tuple[str, ...]:
+    blocks: Final = _content_blocks(message)
+    texts: Final = tuple(text for block in blocks if (text := _plain_text(block)) is not None)
+    return texts if len(texts) == len(blocks) else ()
+
+
 def _collapse_single_text_block(message: Mapping[str, object]) -> Mapping[str, object]:
-    content: Final = message.get("content")
-    match content:
-        case [{"type": "text", "text": str() as text}] if content == [{"type": "text", "text": text}]:
+    match _plain_texts(message):
+        case (text,):
             return {**message, "content": text}
         case _:
             return message
 
 
+def _without_text_cache_control(block: object) -> object:
+    match block:
+        case {"type": "text", "text": str() as text, "cache_control": object() as cache_control} if block == {
+            "type": "text",
+            "text": text,
+            "cache_control": cache_control,
+        }:
+            return {"type": "text", "text": text}
+        case _:
+            return block
+
+
+def _without_cache_control(message: AllMessageValues) -> AllMessageValues:
+    uncached: Final = {key: value for key, value in message.items() if key != "cache_control"}
+    match content := uncached.get("content"):
+        case list():
+            blocks: Final = [_without_text_cache_control(block) for block in _CONTENT_BLOCKS.validate_python(content)]
+            return cast(AllMessageValues, {**uncached, "content": blocks})  # cast-ok: spread keeps the shape
+        case _:
+            return cast(AllMessageValues, uncached)  # cast-ok: only cache_control was left out
+
+
+def _has_string_content(message: Mapping[str, object]) -> bool:
+    return isinstance(message.get("content"), str)
+
+
+def _is_system(message: Mapping[str, object]) -> bool:
+    return message.get("role") == "system"
+
+
+def _json_mode_message(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
+    candidate: Final = _without_cache_control(message) if drop_params else message
+    texts: Final = _plain_texts(candidate)
+    if "cache_control" in candidate or not (texts or _has_string_content(candidate)):
+        return message
+    if sum(1 for text in texts if text.strip()) < 2:
+        return candidate
+    return cast(AllMessageValues, {**candidate, "content": "".join(texts)})  # cast-ok: spread keeps the shape
+
+
+def _is_dropped_on_send(message: Mapping[str, object]) -> bool:
+    sanitized: Final = {**message}
+    _sanitize_empty_content(sanitized)
+    return _is_bare_assistant_message(sanitized)
+
+
+def _is_system_or_dropped(message: Mapping[str, object]) -> bool:
+    return _is_system(message) or _is_dropped_on_send(message)
+
+
+def _json_checked_start(messages: Sequence[AllMessageValues]) -> int:
+    trailing: Final = tuple(takewhile(_is_system_or_dropped, reversed(messages)))
+    last_sent_offset: Final = 0 if any(_is_system(message) for message in trailing) else 1
+    return max(len(messages) - len(trailing) - last_sent_offset, 0)
+
+
+def _json_object_ready_messages(
+    model: str,
+    messages: list[AllMessageValues],  # mutable-ok: inherited transform_request contract
+    response_format: object,
+    drop_params: bool,
+) -> list[AllMessageValues]:  # mutable-ok: inherited transform_request contract
+    match response_format:
+        case {"type": "json_object"} if "claude" not in model and model.startswith(DATABRICKS_HOSTED_MODEL_PREFIXES):
+            checked_start: Final = _json_checked_start(messages)
+            return [
+                *messages[:checked_start],
+                *(_json_mode_message(message, drop_params) for message in messages[checked_start:]),
+            ]
+        case _:
+            return messages
+
+
+def _is_empty_text_block(block: object) -> bool:
+    match block:
+        case {"type": "text", "text": str() as text}:
+            return not text.strip()
+        case {"type": "text", "text": object() as text}:
+            return not text
+        case {"type": "text"}:
+            return True
+        case _:
+            return False
+
+
+def _non_string_text_type(block: object) -> str | None:
+    match block:
+        case {"type": "text", "text": str()}:
+            return None
+        case {"type": "text", "text": object() as text} if text:
+            return type(text).__name__
+        case _:
+            return None
+
+
+def _first_non_string_text_type(messages: Sequence[AllMessageValues]) -> str | None:
+    blocks: Final = chain.from_iterable(map(_content_blocks, messages))
+    return next((type_name for block in blocks if (type_name := _non_string_text_type(block))), None)
+
+
 def _sanitize_empty_content(message_dict: dict[str, object]) -> None:
     """
     Remove or filter content so empty text blocks are not sent.
@@ -108,9 +235,7 @@ def _sanitize_empty_content(message_dict: dict[str, object]) -> None:
             message_dict.pop("content")
             return
         filtered: Final = [
-            block
-            for block in content
-            if not (isinstance(block, dict) and block.get("type") == "text" and not (block.get("text") or "").strip())
+            block for block in _CONTENT_BLOCKS.validate_python(content) if not _is_empty_text_block(block)
         ]
         if not filtered:
             message_dict.pop("content")
@@ -343,7 +468,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         use_ai_gateway: Final = self.uses_ai_gateway(model)
         return join_api_base(self._get_api_base(api_base, use_ai_gateway=use_ai_gateway), "chat/completions")
 
-    def get_supported_openai_params(self, model: str | None = None) -> list:
+    def get_supported_openai_params(self, model: str | None = None) -> list[str]:
         return [
             "stream",
             "stop",
@@ -449,8 +574,13 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         drop_params: bool,
         replace_max_completion_tokens_with_max_tokens: bool = True,
     ) -> dict:
-        is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params)
-        mapped_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
+        request_params: Final = (
+            tool_params_without_hosted_mcp(non_default_params, model=model, route="databricks", drop_params=drop_params)
+            if "claude" in model
+            else non_default_params
+        )
+        is_thinking_enabled: Final = self.is_thinking_enabled(request_params)
+        mapped_params: Final = super().map_openai_params(request_params, optional_params, model, drop_params)
         if "claude" in model:
             AnthropicConfig.translate_legacy_thinking_for_adaptive_model(
                 model=model, optional_params=mapped_params, custom_llm_provider="databricks", drop_params=drop_params
@@ -459,16 +589,16 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
             mapped_params["tools"] = self._map_openai_to_dbrx_tool(
                 model=model, tools=mapped_params["tools"], drop_params=drop_params
             )
-        if "max_completion_tokens" in non_default_params and replace_max_completion_tokens_with_max_tokens:
-            mapped_params["max_tokens"] = non_default_params[
+        if "max_completion_tokens" in request_params and replace_max_completion_tokens_with_max_tokens:
+            mapped_params["max_tokens"] = request_params[
                 "max_completion_tokens"
             ]  # most openai-compatible providers support 'max_tokens' not 'max_completion_tokens'
             mapped_params.pop("max_completion_tokens", None)
 
-        if "response_format" in non_default_params and "claude" in model:
+        if "response_format" in request_params and "claude" in model:
             _tool: Final = self.map_response_format_to_databricks_tool(
                 model,
-                non_default_params["response_format"],
+                request_params["response_format"],
                 mapped_params,
                 is_thinking_enabled,
             )
@@ -486,8 +616,8 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
                 "response_format", None
             )  # unsupported for claude models - if json_schema -> convert to tool call
 
-        if "reasoning_effort" in non_default_params and self._uses_anthropic_thinking_param(model):
-            reasoning_effort_value: Final = non_default_params.get("reasoning_effort")
+        if "reasoning_effort" in request_params and self._uses_anthropic_thinking_param(model):
+            reasoning_effort_value: Final = request_params.get("reasoning_effort")
             mapped_thinking: Final = AnthropicConfig._map_reasoning_effort(
                 reasoning_effort=reasoning_effort_value,
                 model=model,
@@ -513,7 +643,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
             optional_params.pop("reasoning_effort", None)
         ## handle thinking tokens
         self.update_optional_params_with_thinking_tokens(
-            non_default_params=non_default_params, optional_params=mapped_params
+            non_default_params=request_params, optional_params=mapped_params
         )
         gated_thinking: Final = strip_beta_gated_thinking(
             mapped_params.get("thinking"),  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # mapped_params is the base config's bare dict
@@ -535,6 +665,35 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
 
         return False
 
+    def transform_request(
+        self,
+        model: str,
+        messages: list[AllMessageValues],  # mutable-ok: inherited contract
+        optional_params: dict[str, object],  # mutable-ok: inherited contract
+        litellm_params: dict[str, object],  # mutable-ok: inherited contract
+        headers: dict[str, object],  # mutable-ok: inherited contract
+    ) -> dict[str, object]:  # mutable-ok: inherited contract
+        match _first_non_string_text_type(messages):
+            case str() as type_name:
+                raise DatabricksException(
+                    status_code=400,
+                    message=f"A text content block needs a string 'text', got {type_name}",
+                )
+            case None:
+                pass
+        return super().transform_request(
+            model=model,
+            messages=_json_object_ready_messages(
+                model,
+                messages,
+                optional_params.get("response_format"),
+                drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
+            ),
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
     @overload
     def _transform_messages(
         self, messages: list[AllMessageValues], model: str, is_async: Literal[True]

litellm/llms/databricks/common_utils.py

@@ -56,6 +56,7 @@ DATABRICKS_WORKSPACE_ROUTE_PREFIXES: Final = (
     DATABRICKS_AI_GATEWAY_MESSAGES_PATH,
 )
 DATABRICKS_NATIVE_MESSAGES_MODEL_PREFIXES: Final = ("databricks-claude-", "system.ai.claude-")
+DATABRICKS_HOSTED_MODEL_PREFIXES: Final = ("databricks-", "system.ai.")
 DATABRICKS_ENDPOINT_PATHS: Final = MappingProxyType(
     {
         "chat_completions": "/chat/completions",

litellm/llms/deepseek/chat/transformation.py

@@ -12,6 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_search_results_text,
 )
 from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
+from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
@@ -33,17 +34,15 @@ def _warn_on_dropped_budget(thinking: object, model: str) -> None:
 
 
 class DeepSeekChatConfig(OpenAIGPTConfig):
-    def get_supported_openai_params(self, model: str) -> list:
+    def get_supported_openai_params(self, model: str) -> list[str]:
         """
         DeepSeek reasoner models support thinking parameter.
         """
-        params: Final = super().get_supported_openai_params(model)
-        params.extend(["thinking", "reasoning_effort"])
-        return params
+        return [*super().get_supported_openai_params(model), "thinking", "reasoning_effort"]
 
     def map_openai_params(
         self,
-        non_default_params: dict,
+        non_default_params: dict[str, object],
         optional_params: dict,
         model: str,
         drop_params: bool,
@@ -59,7 +58,12 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
         Reference: https://api-docs.deepseek.com/guides/thinking_mode
         """
         # Let parent handle standard params first
-        optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params)
+        optional_params = super().map_openai_params(
+            tool_params_without_hosted_mcp(non_default_params, model=model, route="deepseek", drop_params=drop_params),
+            optional_params,
+            model,
+            drop_params,
+        )
 
         # Pop thinking/reasoning_effort from optional_params first (parent may have added them)
         # Then re-add only if valid for DeepSeek
@@ -265,6 +269,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
         tools: Final = optional_params.get("tools")
         if not isinstance(tools, list) or not tools:
             return optional_params
+        sent_tools: Final[tuple[object, ...]] = tuple(tools)
 
         def _is_function_tool(tool: object) -> bool:
             return isinstance(tool, dict) and tool.get("type") == "function"
@@ -289,14 +294,14 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
             name: Final = function.get("name")
             return isinstance(name, str) and name in function_tool_names
 
-        function_tools: Final = [tool for tool in tools if _is_function_tool(tool)]
-        if len(function_tools) == len(tools):
+        function_tools: Final = [tool for tool in sent_tools if _is_function_tool(tool)]
+        if len(function_tools) == len(sent_tools):
             return optional_params
 
         dropped_types: Final = sorted(
             {
                 str(tool.get("type")) if isinstance(tool, dict) else type(tool).__name__
-                for tool in tools
+                for tool in sent_tools
                 if not _is_function_tool(tool)
             }
         )

litellm/llms/fireworks_ai/responses/transformation.py

@@ -29,6 +29,7 @@ from litellm.utils import (
     declared_mid_conversation_system_support,
     deployment_declared_mid_conversation_system_support,
     deployment_model_info,
+    get_bundled_model_cost_map,
 )
 
 if TYPE_CHECKING:
@@ -54,6 +55,15 @@ def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str,
     )
 
 
+def _bundled_cost_map_declares_mid_conversation_system(model: str) -> bool:
+    bundled_cost_map: Final[Mapping[str, object]] = get_bundled_model_cost_map()
+    names: Final = (model.removeprefix("fireworks_ai/"), resolve_fireworks_resource_name(model))
+    return any(
+        deployment_declared_mid_conversation_system_support(bundled_cost_map.get(f"fireworks_ai/{name}")) is True
+        for name in names
+    )
+
+
 class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     @property
     def custom_llm_provider(self) -> LlmProviders:
@@ -90,9 +100,13 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         validated_input: Final = self._validate_input_param(input)
         model_info: Final = deployment_model_info(litellm_params)
         declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info)
+        keeps_developer_items_in_place: Final = declared is True and (
+            deployment_declared_mid_conversation_system_support(model_info) is True
+            or _bundled_cost_map_declares_mid_conversation_system(model)
+        )
         folded: Final = (
             None
-            if deployment_declared_mid_conversation_system_support(model_info) is True
+            if keeps_developer_items_in_place
             else instruction_items_folded(
                 validated_input, response_api_optional_request_params, "user" if declared is False else "system"
             )

litellm/llms/gemini/chat/transformation.py

@@ -1,5 +1,5 @@
 from collections.abc import Mapping
-from typing import Final, cast
+from typing import ClassVar, Final, cast
 
 import litellm
 from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -49,6 +49,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
     Note: Please make sure to modify the default parameters as required for your use case.
     """
 
+    _route: ClassVar[str] = "gemini"
+
     temperature: float | None = None
     max_output_tokens: int | None = None
     top_p: float | None = None

litellm/llms/gemini/common_utils.py

@@ -537,13 +537,16 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
     ) -> TokenCountResponse | None:
         import copy
 
+        from litellm.llms.anthropic.messages.context_management.inference_view import (
+            InferenceRequestViewError,
+            count_context_edits,
+            inference_request_view,
+        )
         from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter
         from litellm.llms.vertex_ai.count_tokens.context import (
             VertexGeminiCountContextError,
-            count_context_edits,
             count_context_error_response,
             gemini_count_contents,
-            inference_request_view,
             malformed_tool_result_text_count_error,
             provider_error_response,
             unconvertible_messages_error,
@@ -556,7 +559,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
             messages_format,
             count_context_edits(context_management, tools, (deployment or {}).get("litellm_params", {}), model_to_use),
         )
-        if isinstance(request_view, VertexGeminiCountContextError):
+        if isinstance(request_view, InferenceRequestViewError):
             return count_context_error_response(request_view, request_model, model_to_use)
         count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools, model=model_to_use)
         if isinstance(count_context, VertexGeminiCountContextError):

litellm/llms/gigachat/chat/transformation.py

@@ -17,6 +17,7 @@ import httpx
 from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.gigachat.utils import convert_usage, get_api_base
@@ -169,7 +170,10 @@ class GigaChatConfig(BaseConfig):
         drop_params: bool,
     ) -> dict:  # mutable-ok: base class contract returns dict
         """Map OpenAI parameters to GigaChat parameters."""
-        for param, value in non_default_params.items():
+        request_params: Final = tool_params_without_hosted_mcp(
+            dict(non_default_params), model=model, route="gigachat", drop_params=drop_params
+        )
+        for param, value in request_params.items():
             if param == "stream":
                 optional_params["stream"] = value
             elif param == "temperature":

litellm/llms/ollama/chat/transformation.py

@@ -4,7 +4,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final, cast
 
 from httpx._models import Headers, Response
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 
 import litellm
 from litellm._uuid import uuid
@@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_reasoning_content,
 )
 from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
+from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@@ -40,6 +41,8 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
+_REQUEST_PARAMS: Final = TypeAdapter(dict[str, object])
+
 
 class OllamaChatConfig(BaseConfig):
     """
@@ -152,7 +155,13 @@ class OllamaChatConfig(BaseConfig):
         model: str,
         drop_params: bool,
     ) -> dict:
-        for param, value in non_default_params.items():
+        request_params: Final = tool_params_without_hosted_mcp(
+            _REQUEST_PARAMS.validate_python(non_default_params),
+            model=model,
+            route="ollama_chat",
+            drop_params=drop_params,
+        )
+        for param, value in request_params.items():
             if param == "max_tokens" or param == "max_completion_tokens":
                 optional_params["num_predict"] = value
             if param == "stream":

litellm/llms/openai/chat/gpt_transformation.py

@@ -21,6 +21,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
     should_convert_tool_call_to_json_mode,
 )
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    chat_completion_prompt_cache_breakpoints_for_wire,
     drop_non_python_regex_patterns,
     drop_tool_reference_parts_from_tool_messages,
     flatten_combinators_and_drop_non_python_regex_patterns,
@@ -556,9 +557,10 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         """
         A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
         api.openai.com, or one opted in through ``prompt_cache_options`` or the cost
-        map) gets each Anthropic ``cache_control`` mark rewritten in place. Any other
-        endpoint keeps or strips the marks as before, and under drop_params also loses
-        every ``prompt_cache_breakpoint`` it cannot read.
+        map) gets each Anthropic ``cache_control`` mark rewritten in place, and under
+        drop_params each content part's ``prompt_cache_breakpoint`` cut down to what chat
+        completions accepts. Any other endpoint keeps or strips the marks as before, and
+        under drop_params also loses every ``prompt_cache_breakpoint`` it cannot read.
         """
         from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
 
@@ -572,9 +574,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
                 _claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
                 tools,
             )
-        drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
+        drop_params: Final = _drops_params(litellm_params)
+        drops_breakpoints: Final = not reads_breakpoints and drop_params
         renamed: Final = (
-            [AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
+            [
+                AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(
+                    chat_completion_prompt_cache_breakpoints_for_wire(message, drop_params)
+                )
+                for message in messages
+            ]
             if reads_breakpoints
             else [without_prompt_cache_breakpoints(message) for message in messages]
             if drops_breakpoints

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -1988,7 +1988,7 @@ class OpenAIResponsesHandler(BaseTranslation):
             "annotations": (),
             "logprobs": None,
         }
-        return (
+        events: Final = (
             *_open_item_closing_events(responses_so_far),
             OutputItemAddedEvent(
                 type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
@@ -2033,6 +2033,10 @@ class OpenAIResponsesHandler(BaseTranslation):
                 response=_blocked_response(exc, response_id=response_id, model=model, output_item=item),
             ),
         )
+        return tuple(
+            event.model_copy(update={"sequence_number": number})
+            for number, event in enumerate(events, start=_next_stream_sequence_number(responses_so_far))
+        )
 
 
 class _BlockedContentPart(TypedDict):

litellm/llms/openai/responses/transformation.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
-from functools import lru_cache
+from functools import lru_cache, partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints
 
@@ -12,7 +12,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.integrations.anthropic_cache_control_hook import (
     AnthropicCacheControlHook,
-    openai_prompt_cache_breakpoint_support,
+    deployment_prompt_cache_breakpoint_support,
     targets_openai_api,
 )
 from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers
@@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     drop_non_python_regex_patterns,
     flatten_combinators_and_drop_non_python_regex_patterns,
     prompt_cache_breakpoint_for_wire,
+    whole_prompt_cache_breakpoint_or_none,
     with_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@@ -103,13 +104,6 @@ _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str,
 _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]])
 
 
-class _DeclaredModel(BaseModel):
-    model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
-
-    base_model: object = None
-    model_info: object = None
-
-
 class _DeploymentHost(BaseModel):
     """`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`."""
 
@@ -241,12 +235,12 @@ def _json_array_or_empty(value: object) -> Sequence[object]:
     return value if isinstance(value, (list, tuple)) else ()
 
 
-def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object:
+def _item_with_wire_safe_prompt_cache_breakpoints(item: object, to_wire: Callable[[object], object] | None) -> object:
     fields: Final = _json_object_or_none(item)
     if fields is None or not _carries_prompt_cache_breakpoint(fields):
         return item
     wire_safe_fields: Final = {
-        key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers)
+        key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, to_wire)
         for key, value in fields.items()
         if key != "prompt_cache_breakpoint"
     }
@@ -270,21 +264,23 @@ def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool:
     return any(isinstance(part, Mapping) and "prompt_cache_breakpoint" in part for part in _json_array_or_empty(parts))
 
 
-def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object:
+def _parts_with_wire_safe_prompt_cache_breakpoints(
+    key: str, value: object, to_wire: Callable[[object], object] | None
+) -> object:
     if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value):
         return value
     parts: Final = _json_array_or_empty(value)
-    wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in parts)
+    wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, to_wire) for part in parts)
     return value if _same_objects(wire_safe_parts, parts) else list(wire_safe_parts)
 
 
-def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object:
+def _part_with_wire_safe_prompt_cache_breakpoint(part: object, to_wire: Callable[[object], object] | None) -> object:
     fields: Final = _json_object_or_none(part)
     if fields is None or "prompt_cache_breakpoint" not in fields:
         return part
     marker: Final = fields["prompt_cache_breakpoint"]
-    wire_marker: Final = prompt_cache_breakpoint_for_wire(marker, drop_params=True) if keeps_markers else None
-    if keeps_markers and wire_marker == marker:
+    wire_marker: Final = None if to_wire is None else to_wire(marker)
+    if to_wire is not None and wire_marker == marker:
         return part
     unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
     return with_prompt_cache_breakpoint(unmarked, wire_marker)
@@ -681,24 +677,31 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
     ) -> str | ResponseInputParam:
         if not isinstance(input, list) or not drops_params(litellm_params):
             return input
-        keeps_markers: Final = reads_breakpoints or self._keeps_prompt_cache_breakpoints(
-            model=model, litellm_params=litellm_params
-        )
-        wire_safe_items: Final = tuple(
-            _item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input
+        known_support: Final = self._prompt_cache_breakpoint_support(model=model, litellm_params=litellm_params)
+        to_wire: Final = (
+            self._prompt_cache_breakpoint_to_wire(litellm_params, reads_breakpoints, known_support)
+            if reads_breakpoints or known_support is not False
+            else None
         )
+        wire_safe_items: Final = tuple(_item_with_wire_safe_prompt_cache_breakpoints(item, to_wire) for item in input)
         if _same_objects(wire_safe_items, input):
             return input
         wire_safe_input: Final = list(wire_safe_items)
         return cast("ResponseInputParam", wire_safe_input)  # cast-ok: items keep their shape, minus a rejected marker
 
-    def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool:
-        provider: Final = _provider_name(self.custom_llm_provider)
-        base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params)
-        known_support: Final = (
-            openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model))
+    def _prompt_cache_breakpoint_to_wire(
+        self, litellm_params: GenericLiteLLMParams, reads_breakpoints: bool, known_support: bool | None
+    ) -> Callable[[object], object]:
+        opted_in_openai_host: Final = reads_breakpoints and self.custom_llm_provider == LlmProviders.OPENAI
+        known_openai_reader: Final = known_support is True and self._targets_openai_api(litellm_params)
+        if opted_in_openai_host or known_openai_reader:
+            return partial(prompt_cache_breakpoint_for_wire, drop_params=True)
+        return whole_prompt_cache_breakpoint_or_none
+
+    def _prompt_cache_breakpoint_support(self, model: str, litellm_params: GenericLiteLLMParams) -> bool | None:
+        return deployment_prompt_cache_breakpoint_support(
+            model, _provider_name(self.custom_llm_provider), litellm_params
         )
-        return next((support for support in known_support if support is not None), None) is not False
 
     def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool:
         host: Final = _DeploymentHost.model_validate(litellm_params)
@@ -872,13 +875,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None
         return base_model if isinstance(base_model, str) and base_model else model
 
-    @staticmethod
-    def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str:
-        declared: Final = _DeclaredModel.model_validate(litellm_params)
-        model_info: Final = _json_object_or_none(declared.model_info)
-        candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model"))
-        return next((name for name in candidates if isinstance(name, str) and name), model)
-
     @staticmethod
     def _sanitized_tool_entry(
         entry: Mapping[str, object],

litellm/llms/opencode/chat/transformation.py

@@ -1,5 +1,5 @@
 from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
-from typing import TYPE_CHECKING, Final
+from typing import TYPE_CHECKING, ClassVar, Final
 
 import httpx
 from pydantic import TypeAdapter
@@ -159,6 +159,7 @@ class OpenCodeGoMessagesChatConfig(_GoReportedCostMixin, AnthropicConfig):
 
 class OpenCodeGoGeminiChatConfig(_GoReportedCostMixin, GoogleAIStudioGeminiConfig):
     go_usage_key: str = "usageMetadata"
+    _route: ClassVar[str] = "opencode_go"
 
     @property
     def supports_stream_param_in_request_body(self) -> bool:

litellm/llms/vertex_ai/common_utils.py

@@ -1458,6 +1458,7 @@ class VertexAITokenCounter(BaseTokenCounter):
     ) -> TokenCountResponse | None:
         import copy
 
+        from litellm.llms.anthropic.common_utils import context_management_with_edits
         from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_publisher
         from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
             VertexAIError as PartnerVertexAIError,
@@ -1539,6 +1540,9 @@ class VertexAITokenCounter(BaseTokenCounter):
                     system=system,
                     tools=tools,
                     count_options=count_options,
+                    context_management=(
+                        None if context_management is None else context_management_with_edits(context_management.spec)
+                    ),
                 )
             except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
                 status_code: Final = e.response.status_code
@@ -1565,12 +1569,15 @@ class VertexAITokenCounter(BaseTokenCounter):
                     original_response=result,
                 )
         else:
+            from litellm.llms.anthropic.messages.context_management.inference_view import (
+                InferenceRequestViewError,
+                count_context_edits,
+                inference_request_view,
+            )
             from litellm.llms.vertex_ai.count_tokens.context import (
                 VertexGeminiCountContextError,
-                count_context_edits,
                 count_context_error_response,
                 gemini_count_contents,
-                inference_request_view,
                 malformed_tool_result_text_count_error,
                 provider_error_response,
                 unconvertible_messages_error,
@@ -1584,7 +1591,7 @@ class VertexAITokenCounter(BaseTokenCounter):
                 messages_format,
                 count_context_edits(context_management, tools, count_tokens_params_request, model_to_use),
             )
-            if isinstance(request_view, VertexGeminiCountContextError):
+            if isinstance(request_view, InferenceRequestViewError):
                 return count_context_error_response(request_view, request_model, model_to_use)
             count_context: Final = vertex_gemini_count_context(
                 system=request_view.system, tools=tools, model=model_to_use

litellm/llms/vertex_ai/context_caching/rejected_cache_retry.py

@@ -1,4 +1,4 @@
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Mapping, Sequence
 from copy import deepcopy
 from functools import partial
 from typing import Final, TypeVar
@@ -17,16 +17,27 @@ Built: Final = TypeVar("Built")
 Sent: Final = TypeVar("Sent")
 
 
-def cache_rebuild(
+def _build_from_copies(
+    build: Callable[..., Built], messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
+) -> Built:
+    return build(messages=deepcopy(list(messages)), optional_params=independent_snapshot(dict(optional_params)))
+
+
+def cache_rebuilds(
     build: Callable[..., Built], messages: list[AllMessageValues], optional_params: dict
-) -> Callable[[], Built] | None:
-    """Copies the inputs before the first transform, which pops keys from ``optional_params`` and ``messages``."""
+) -> tuple[Callable[[], Built], Callable[[], Built]] | tuple[None, None]:
+    """Returns one build with a fresh cache and one with the history inline, both over a snapshot of the inputs
+    taken before the first transform, which pops keys from ``optional_params`` and ``messages``. Each call
+    copies the snapshot again, since every transform consumes the inputs it is given."""
     if not any(map(is_cached_message, messages)):
-        return None
-    return partial(build, messages=deepcopy(messages), optional_params=independent_snapshot(optional_params))
+        return None, None
+    from_snapshot: Final = partial(
+        _build_from_copies, messages=deepcopy(messages), optional_params=independent_snapshot(optional_params)
+    )
+    return partial(from_snapshot, build), partial(from_snapshot, partial(build, context_caching=False))
 
 
-def _status_and_message(error: httpx.HTTPStatusError | BaseLLMException) -> tuple[int, str]:
+def status_and_message(error: httpx.HTTPStatusError | BaseLLMException) -> tuple[int, str]:
     match error:
         case httpx.HTTPStatusError():
             return error.response.status_code, error.response.text
@@ -35,7 +46,7 @@ def _status_and_message(error: httpx.HTTPStatusError | BaseLLMException) -> tupl
 
 
 def _cache_rejected(body: RequestBody, error: httpx.HTTPStatusError | BaseLLMException) -> bool:
-    return body.get("cachedContent") is not None and names_rejected_cache(*_status_and_message(error))
+    return body.get("cachedContent") is not None and names_rejected_cache(*status_and_message(error))
 
 
 async def post_with_fresh_cache_async(

litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py

@@ -425,6 +425,7 @@ class ContextCachingEndpoints(VertexBase):
         vertex_auth_header: str | None,
         extra_headers: dict | None = None,
         cached_content: str | None = None,
+        context_caching: bool = True,
     ) -> tuple[list[AllMessageValues], dict, str | None]:
         """
         Receives
@@ -436,7 +437,7 @@ class ContextCachingEndpoints(VertexBase):
 
         Follows - https://ai.google.dev/api/caching#request-body
         """
-        if cached_content is not None:
+        if cached_content is not None or not context_caching:
             return messages, optional_params, cached_content
 
         cached_messages, non_cached_messages = separate_cached_messages(messages=messages)
@@ -573,6 +574,7 @@ class ContextCachingEndpoints(VertexBase):
         vertex_auth_header: str | None,
         extra_headers: dict | None = None,
         cached_content: str | None = None,
+        context_caching: bool = True,
     ) -> tuple[list[AllMessageValues], dict, str | None]:
         """
         Receives
@@ -584,7 +586,7 @@ class ContextCachingEndpoints(VertexBase):
 
         Follows - https://ai.google.dev/api/caching#request-body
         """
-        if cached_content is not None:
+        if cached_content is not None or not context_caching:
             return messages, optional_params, cached_content
 
         cached_messages, non_cached_messages = separate_cached_messages(messages=messages)

litellm/llms/vertex_ai/count_tokens/context.py

@@ -11,10 +11,11 @@ from litellm.litellm_core_utils.prompt_templates.factory import chat_completion_
 from litellm.types.llms.anthropic import AnthropicMessagesTextParam
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.llms.vertex_ai import ContentType
-from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
+from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
 
 if TYPE_CHECKING:
     from litellm.exceptions import APIError
+    from litellm.llms.anthropic.messages.context_management.inference_view import InferenceRequestViewError
 
 
 @dataclass(frozen=True, slots=True)
@@ -152,7 +153,7 @@ def malformed_tool_result_text_count_error(
 
 
 def count_context_error_response(
-    error: VertexGeminiCountContextError, request_model: str, model_used: str
+    error: "VertexGeminiCountContextError | InferenceRequestViewError", request_model: str, model_used: str
 ) -> TokenCountResponse:
     return TokenCountResponse(
         total_tokens=0,
@@ -178,130 +179,6 @@ def provider_error_response(error: "APIError", request_model: str, model_used: s
     )
 
 
-@dataclass(frozen=True, slots=True)
-class InferenceRequestView:
-    messages: Sequence[Mapping[str, object]]
-    system: object | None
-
-
-@dataclass(frozen=True, slots=True)
-class CountContextEdits:
-    spec: JsonValue
-    model: str
-    tools: tuple[Mapping[str, object], ...] | None
-    drop_invalid_edits: bool = False
-
-
-_CONTEXT_MANAGEMENT_SPEC: Final[TypeAdapter[Mapping[str, object] | tuple[Mapping[str, object], ...]]] = TypeAdapter(
-    Mapping[str, object] | tuple[Mapping[str, object], ...]
-)
-
-
-def count_context_edits(
-    context_management: CountContextManagement | None,
-    tools: Sequence[Mapping[str, object]] | None,
-    litellm_params: Mapping[str, object],
-    fallback_model: str,
-) -> CountContextEdits:
-    model: Final = litellm_params.get("model")
-    return CountContextEdits(
-        spec=None if context_management is None else context_management.spec,
-        model=model if isinstance(model, str) else fallback_model,
-        tools=None if tools is None else tuple(tools),
-        drop_invalid_edits=context_management is not None and context_management.drop_invalid_edits,
-    )
-
-
-def _typed_spec(spec: JsonValue) -> Mapping[str, object] | tuple[Mapping[str, object], ...] | None:
-    try:
-        return _CONTEXT_MANAGEMENT_SPEC.validate_python(spec)
-    except ValidationError:
-        return None
-
-
-def _typed_system(
-    system: object | None,
-) -> str | tuple[AnthropicMessagesTextParam, ...] | None | VertexGeminiCountContextError:
-    if system is None or isinstance(system, str):
-        return system
-    try:
-        return _SYSTEM_BLOCKS.validate_python(system)
-    except ValidationError:
-        return _INVALID_SYSTEM
-
-
-def _view_without_edits(
-    messages: Sequence[Mapping[str, object]],
-    system: object | None,
-    typed_system: str | Sequence[Mapping[str, object]] | None,
-) -> InferenceRequestView:
-    from litellm.llms.anthropic.messages.context_management.editors.compact import (
-        apply_client_compaction_block_history,
-    )
-
-    history: Final = apply_client_compaction_block_history(
-        messages=[dict(message) for message in messages],
-        system=typed_system
-        if typed_system is None or isinstance(typed_system, str)
-        else [dict(block) for block in typed_system],
-    )
-    if history is None:
-        return InferenceRequestView(messages=messages, system=system)
-    return InferenceRequestView(messages=history.messages, system=history.system)
-
-
-async def _anthropic_inference_view(
-    messages: Sequence[Mapping[str, object]], system: object | None, edits: CountContextEdits
-) -> InferenceRequestView | VertexGeminiCountContextError:
-    from litellm.llms.anthropic.messages.adapters.handler import prepare_context_managed_request
-    from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
-    from litellm.llms.anthropic.messages.native.utils import normalize_native_messages
-
-    typed_system: Final = _typed_system(system)
-    if isinstance(typed_system, VertexGeminiCountContextError):
-        return typed_system
-    normalized: Final = normalize_native_messages(messages)
-    spec: Final = _typed_spec(edits.spec)
-    try:
-        prepared: Final = await prepare_context_managed_request(
-            model=edits.model,
-            messages=normalized,
-            tools=None if edits.tools is None else [dict(tool) for tool in edits.tools],
-            system=[dict(block) for block in typed_system] if isinstance(typed_system, tuple) else typed_system,
-            context_management_spec=(
-                None if spec is None else dict(spec) if isinstance(spec, Mapping) else [dict(edit) for edit in spec]
-            ),
-            litellm_metadata=None,
-            additional_drop_params=None,
-            llm_router=None,
-            count_only=True,
-            drop_invalid_edits=edits.drop_invalid_edits,
-        )
-    except AnthropicContextManagementError as error:
-        if error.status_code >= 500:
-            return _view_without_edits(normalized, system, typed_system)
-        return VertexGeminiCountContextError(error_message=error.message, status_code=error.status_code)
-    if prepared is None:
-        return InferenceRequestView(messages=normalized, system=system)
-    return InferenceRequestView(messages=prepared.messages, system=prepared.system)
-
-
-async def inference_request_view(
-    messages: Sequence[Mapping[str, object]],
-    system: object | None,
-    messages_format: TokenCountMessagesFormat,
-    edits: CountContextEdits,
-) -> InferenceRequestView | VertexGeminiCountContextError:
-    """Apply the history rewrites and context edits /v1/messages runs before it converts a body for Gemini."""
-    match messages_format:
-        case "openai":
-            return InferenceRequestView(messages=messages, system=system)
-        case "anthropic":
-            return await _anthropic_inference_view(messages, system, edits)
-        case _:
-            assert_never(messages_format)
-
-
 def _chat_messages(
     messages: Sequence[Mapping[str, object]],
     messages_format: TokenCountMessagesFormat,

litellm/llms/vertex_ai/gemini/rejected_thought_signature_retry.py

@@ -0,0 +1,235 @@
+from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from typing import Final, TypeVar
+
+import httpx
+from pydantic import TypeAdapter, ValidationError
+
+from litellm import verbose_logger
+from litellm.litellm_core_utils.prompt_templates.factory import get_dummy_thought_signature
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.vertex_ai import ContentType, PartType, RequestBody
+
+from ..context_caching.rejected_cache_retry import status_and_message
+
+Body: Final = TypeVar("Body", bound=Mapping[str, object])
+Sent: Final = TypeVar("Sent")
+
+_REJECTED_SIGNATURE_MARKER: Final = "thought signature"
+_SIGNATURE_KEYS: Final = frozenset({"thoughtSignature", "thought_signature"})
+_ENTRIES: Final = TypeAdapter(list[dict[str, object]])
+
+
+def names_rejected_thought_signature(status_code: int, message: str) -> bool:
+    """A signature only validates on the backend that minted it. AI Studio answers a replayed Vertex signature
+    with ``400 Corrupted thought signature.`` and Vertex answers an AI Studio one with ``400 Invalid thought
+    signature.``"""
+    return status_code == 400 and _REJECTED_SIGNATURE_MARKER in message.lower()
+
+
+def _is_rejected_signature(error: httpx.HTTPStatusError | BaseLLMException) -> bool:
+    return names_rejected_thought_signature(*status_and_message(error))
+
+
+def _entries(value: object) -> Sequence[Mapping[str, object]]:
+    try:
+        return _ENTRIES.validate_python(value)
+    except ValidationError:
+        return []
+
+
+def _foreign_signatures(body: Mapping[str, object], placeholder: str) -> Iterator[object]:
+    for content in _entries(body.get("contents")):
+        for part in _entries(content.get("parts")):
+            yield from (
+                value for key, value in part.items() if key in _SIGNATURE_KEYS and value and value != placeholder
+            )
+
+
+def _foreign_signature_count(body: Mapping[str, object]) -> int:
+    return sum(1 for _ in _foreign_signatures(body, get_dummy_thought_signature()))
+
+
+def _part_with_placeholder(part: PartType, placeholder: str) -> PartType:
+    if not part.get("thoughtSignature"):
+        return part
+    return {**part, "thoughtSignature": placeholder}
+
+
+def _content_with_placeholders(content: ContentType, placeholder: str) -> ContentType:
+    return {**content, "parts": [_part_with_placeholder(part, placeholder) for part in content["parts"]]}
+
+
+def with_placeholder_thought_signatures(body: RequestBody) -> RequestBody | None:
+    """Swaps every thought signature in the history for Google's documented skip-validator placeholder, or
+    returns ``None`` when the body carries nothing the backend could have rejected."""
+    placeholder: Final = get_dummy_thought_signature()
+    if next(_foreign_signatures(body, placeholder), None) is None:
+        return None
+    return {**body, "contents": [_content_with_placeholders(content, placeholder) for content in body["contents"]]}
+
+
+def _native_part_with_placeholder(part: Mapping[str, object], placeholder: str) -> Mapping[str, object]:
+    return {key: placeholder if key in _SIGNATURE_KEYS and value else value for key, value in part.items()}
+
+
+def _native_content_with_placeholders(content: Mapping[str, object], placeholder: str) -> Mapping[str, object]:
+    parts: Final = _entries(content.get("parts"))
+    if not parts:
+        return content
+    return {**content, "parts": [_native_part_with_placeholder(part, placeholder) for part in parts]}
+
+
+def with_placeholder_native_thought_signatures(body: Mapping[str, object]) -> Mapping[str, object] | None:
+    """The same swap on a ``generateContent`` body as the client sent it, whose parts may spell the key
+    ``thoughtSignature`` or ``thought_signature``."""
+    placeholder: Final = get_dummy_thought_signature()
+    if next(_foreign_signatures(body, placeholder), None) is None:
+        return None
+    return {
+        **body,
+        "contents": [
+            _native_content_with_placeholders(content, placeholder) for content in _entries(body.get("contents"))
+        ],
+    }
+
+
+async def _inline_history_async(body: Body, uncached: Callable[[], Awaitable[Body]] | None) -> Body | None:
+    if uncached is None or body.get("cachedContent") is None:
+        return None
+    inline: Final = await uncached()
+    return inline if _foreign_signature_count(inline) > _foreign_signature_count(body) else None
+
+
+def _inline_history_sync(body: Body, uncached: Callable[[], Body] | None) -> Body | None:
+    if uncached is None or body.get("cachedContent") is None:
+        return None
+    inline: Final = uncached()
+    return inline if _foreign_signature_count(inline) > _foreign_signature_count(body) else None
+
+
+def _resend_body(signed: Body, placeholders: Callable[[Body], Body | None]) -> Body | None:
+    resend_body: Final = placeholders(signed)
+    if resend_body is not None:
+        verbose_logger.warning(
+            "Gemini rejected a replayed thought signature, resending with the skip_thought_signature_validator"
+            " placeholder in place of the replayed signatures"
+        )
+    return resend_body
+
+
+async def build_with_accepted_thought_signatures_async(
+    build: Callable[[], Awaitable[Body]],
+    placeholders: Callable[[Body], Body | None],
+    uncached: Callable[[], Awaitable[Body]] | None,
+) -> Body:
+    """Vertex validates the signatures in a cache's contents when it creates the cache, so a rejected one there
+    sends the history inline with the placeholder instead."""
+    try:
+        return await build()
+    except BaseLLMException as error:
+        if uncached is None or not _is_rejected_signature(error):
+            raise
+        inline_body: Final = _resend_body(await uncached(), placeholders)
+        if inline_body is None:
+            raise
+    return inline_body
+
+
+def build_with_accepted_thought_signatures_sync(
+    build: Callable[[], Body],
+    placeholders: Callable[[Body], Body | None],
+    uncached: Callable[[], Body] | None,
+) -> Body:
+    try:
+        return build()
+    except BaseLLMException as error:
+        if uncached is None or not _is_rejected_signature(error):
+            raise
+        inline_body: Final = _resend_body(uncached(), placeholders)
+        if inline_body is None:
+            raise
+    return inline_body
+
+
+async def _post_inline_async(
+    body: Body,
+    placeholders: Callable[[Body], Body | None],
+    uncached: Callable[[], Awaitable[Body]] | None,
+    post: Callable[[Body], Awaitable[Sent]],
+    log_request: Callable[[Body], None],
+    rejection: httpx.HTTPStatusError | BaseLLMException,
+) -> tuple[Body, Sent]:
+    inline: Final = await _inline_history_async(body, uncached)
+    resend_body: Final = None if inline is None else _resend_body(inline, placeholders)
+    if resend_body is None:
+        raise rejection
+    log_request(resend_body)
+    return resend_body, await post(resend_body)
+
+
+def _post_inline_sync(
+    body: Body,
+    placeholders: Callable[[Body], Body | None],
+    uncached: Callable[[], Body] | None,
+    post: Callable[[Body], Sent],
+    log_request: Callable[[Body], None],
+    rejection: httpx.HTTPStatusError | BaseLLMException,
+) -> tuple[Body, Sent]:
+    inline: Final = _inline_history_sync(body, uncached)
+    resend_body: Final = None if inline is None else _resend_body(inline, placeholders)
+    if resend_body is None:
+        raise rejection
+    log_request(resend_body)
+    return resend_body, post(resend_body)
+
+
+async def post_with_accepted_thought_signatures_async(
+    body: Body,
+    placeholders: Callable[[Body], Body | None],
+    uncached: Callable[[], Awaitable[Body]] | None,
+    post: Callable[[Body], Awaitable[Sent]],
+    log_request: Callable[[Body], None],
+) -> tuple[Body, Sent]:
+    """Innermost retry around the raw post, so a cache rebuild (which re-transforms the messages and brings
+    the rejected signatures back) is itself wrapped by it. The resend keeps the cache and swaps the signatures
+    the body carries, and sends the history inline only when that is rejected too and the cache holds one.
+    Returns the body the backend accepted."""
+    try:
+        return body, await post(body)
+    except (httpx.HTTPStatusError, BaseLLMException) as error:
+        if not _is_rejected_signature(error):
+            raise
+        resend_body: Final = _resend_body(body, placeholders)
+        if resend_body is None:
+            return await _post_inline_async(body, placeholders, uncached, post, log_request, error)
+    log_request(resend_body)
+    try:
+        return resend_body, await post(resend_body)
+    except (httpx.HTTPStatusError, BaseLLMException) as error:
+        if not _is_rejected_signature(error):
+            raise
+        return await _post_inline_async(body, placeholders, uncached, post, log_request, error)
+
+
+def post_with_accepted_thought_signatures_sync(
+    body: Body,
+    placeholders: Callable[[Body], Body | None],
+    uncached: Callable[[], Body] | None,
+    post: Callable[[Body], Sent],
+    log_request: Callable[[Body], None],
+) -> tuple[Body, Sent]:
+    try:
+        return body, post(body)
+    except (httpx.HTTPStatusError, BaseLLMException) as error:
+        if not _is_rejected_signature(error):
+            raise
+        resend_body: Final = _resend_body(body, placeholders)
+        if resend_body is None:
+            return _post_inline_sync(body, placeholders, uncached, post, log_request, error)
+    log_request(resend_body)
+    try:
+        return resend_body, post(resend_body)
+    except (httpx.HTTPStatusError, BaseLLMException) as error:
+        if not _is_rejected_signature(error):
+            raise
+        return _post_inline_sync(body, placeholders, uncached, post, log_request, error)

litellm/llms/vertex_ai/gemini/transformation.py

@@ -1256,6 +1256,7 @@ def sync_transform_request_body(
     vertex_project: str | None,
     vertex_location: str | None,
     vertex_auth_header: str | None,
+    context_caching: bool = True,
 ) -> RequestBody:
     from ..context_caching.vertex_ai_context_caching import ContextCachingEndpoints
 
@@ -1275,6 +1276,7 @@ def sync_transform_request_body(
         timeout=timeout,
         extra_headers=extra_headers,
         cached_content=optional_params.pop("cached_content", None),
+        context_caching=context_caching,
         logging_obj=logging_obj,
         custom_llm_provider=custom_llm_provider,
         vertex_project=vertex_project,
@@ -1390,6 +1392,7 @@ async def async_transform_request_body(
     vertex_project: str | None,
     vertex_location: str | None,
     vertex_auth_header: str | None,
+    context_caching: bool = True,
 ) -> RequestBody:
     from ..context_caching.vertex_ai_context_caching import ContextCachingEndpoints
 
@@ -1409,6 +1412,7 @@ async def async_transform_request_body(
         timeout=timeout,
         extra_headers=extra_headers,
         cached_content=optional_params.pop("cached_content", None),
+        context_caching=context_caching,
         logging_obj=logging_obj,
         custom_llm_provider=custom_llm_provider,
         vertex_project=vertex_project,

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -9,7 +9,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence
 from copy import deepcopy
 from functools import lru_cache, partial
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, Union, cast, get_args
 
 import httpx
 from pydantic import JsonValue, TypeAdapter
@@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc
 from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
 from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
 from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
+from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -103,7 +104,7 @@ from ..common_utils import (
 )
 from ..context_caching.known_cached_contents import known_cached_contents
 from ..context_caching.rejected_cache_retry import (
-    cache_rebuild,
+    cache_rebuilds,
     post_with_fresh_cache_async,
     post_with_fresh_cache_sync,
 )
@@ -134,6 +135,13 @@ from .long_decode import (
     with_initial_prompt_tokens,
     with_spans_shifted,
 )
+from .rejected_thought_signature_retry import (
+    build_with_accepted_thought_signatures_async,
+    build_with_accepted_thought_signatures_sync,
+    post_with_accepted_thought_signatures_async,
+    post_with_accepted_thought_signatures_sync,
+    with_placeholder_thought_signatures,
+)
 from .transformation import (
     async_transform_request_body,
     gemini_convert_messages_with_history,
@@ -437,6 +445,8 @@ _NAMED_UNKNOWN_VOICE_KEYS: Final = 10
 
 _NAMED_VOICE_KEY_CHARS: Final = 100
 
+_NAMED_AUDIO_FORMAT_CHARS: Final = 100
+
 
 def _gemini_speech_config_wrapper(value: Mapping[str, object]) -> Mapping[str, object] | None:
     camel_config: Final = as_str_mapping(value.get("speechConfig"))
@@ -473,6 +483,31 @@ def reject_unknown_gemini_tts_voice_keys(
     )
 
 
+def _named_audio_format(audio_format: object) -> str:
+    if isinstance(audio_format, str):
+        return repr(audio_format[:_NAMED_AUDIO_FORMAT_CHARS])
+    return repr(audio_format)[:_NAMED_AUDIO_FORMAT_CHARS]
+
+
+def unsupported_gemini_tts_audio_format(
+    model: str, audio: Mapping[str, object], drop_params: bool, llm_provider: str | None
+) -> UnsupportedParamsError | None:
+    nested_config: Final = _gemini_speech_config_wrapper(audio) or {}
+    audio_format: Final = next(
+        (item for item in (audio.get("format"), nested_config.get("format")) if item not in (None, "pcm16")), None
+    )
+    if audio_format is None or drops_unsupported_params(drop_params):
+        return None
+    return UnsupportedParamsError(
+        message=(
+            f"{model} does not take the audio format {_named_audio_format(audio_format)}. Set format "
+            "to 'pcm16' or leave it out, or set drop_params to drop it."
+        ),
+        model=model,
+        llm_provider=llm_provider,
+    )
+
+
 def normalize_gemini_speech_config(
     value: Mapping[str, object], drop_unknown_fields: bool = False
 ) -> dict[str, object]:  # mutable-ok: provider request serialization requires a concrete dict
@@ -587,6 +622,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
     Note: Please make sure to modify the default parameters as required for your use case.
     """
 
+    _route: ClassVar[str] = "vertex_ai"
+
     temperature: float | None = None
     max_output_tokens: int | None = None
     top_p: float | None = None
@@ -1555,15 +1592,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             languageCode: "en-US",
         }
         """
-        # Validate audio format - Gemini TTS only supports pcm16
-        audio_format: Final = value.get("format")
-        if audio_format is not None and audio_format != "pcm16":
-            raise ValueError(
-                f"Unsupported audio format for Gemini TTS models: {audio_format}. "
-                f"Gemini TTS models only support 'pcm16' format as they return audio data in L16 PCM format. "
-                f"Please set audio format to 'pcm16'."
-            )
-
+        audio_format_error: Final = unsupported_gemini_tts_audio_format(
+            "Gemini TTS", value, drop_unknown_fields, self.custom_llm_provider
+        )
+        if audio_format_error is not None:
+            raise audio_format_error
         camel_speech_config: Final = value.get("speechConfig")
         snake_speech_config: Final = value.get("speech_config")
         speech_config_value: Final = (
@@ -1601,6 +1634,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
     def _map_chat_audio_params(
         self, model: str, value: Mapping[str, object], drop_params: bool
     ) -> dict[str, object]:  # mutable-ok: the public forwarding interface returns a concrete request dictionary
+        audio_format_error: Final = unsupported_gemini_tts_audio_format(
+            model, value, drop_params, self.custom_llm_provider
+        )
+        if audio_format_error is not None:
+            raise audio_format_error
         if _gemini_speech_config_wrapper(value) is not None:
             reject_unknown_gemini_tts_voice_keys(model, value, drop_params, self.custom_llm_provider)
         return self._map_audio_params(value, drop_unknown_fields=drops_unsupported_params(drop_params))
@@ -1639,9 +1677,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         drop_params: bool,
         allow_integer_enums: bool = True,
     ) -> dict:
-        self._apply_include_server_side_tool_invocations(non_default_params, optional_params)
+        request_params: Final = tool_params_without_hosted_mcp(
+            non_default_params, model=model, route=self._route, drop_params=drop_params
+        )
+        self._apply_include_server_side_tool_invocations(request_params, optional_params)
         supports_sampling_params: Final = self.supports_sampling_params(model)
-        for param, value in non_default_params.items():
+        for param, value in request_params.items():
             if param in ("temperature", "top_p", "top_k") and not supports_sampling_params:
                 continue
             if param == "temperature":
@@ -1705,7 +1746,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 if _tool_choice_value is not None:
                     optional_params["tool_choice"] = _tool_choice_value
             elif param == "parallel_tool_calls":
-                tools_list = non_default_params.get("tools", non_default_params.get("functions"))
+                tools_list = request_params.get("tools", request_params.get("functions"))
                 num_tools = len(tools_list) if isinstance(tools_list, list) else 0
                 # Gemini does not support parallel_tool_calls=False with multiple
                 # tools. Drop the param instead of failing — Responses API clients
@@ -3466,23 +3507,17 @@ class VertexLLM(VertexBase):
             litellm_params=litellm_params,
         )
 
-        rebuild: Final = cache_rebuild(
-            partial(
-                async_transform_request_body,
-                **data,
-                vertex_project=vertex_project,
-                vertex_location=vertex_location,
-                vertex_auth_header=auth_header,
-            ),
-            messages,
-            optional_params,
-        )
-        request_body: Final = await async_transform_request_body(
+        transform: Final = partial(
+            async_transform_request_body,
             **data,
             vertex_project=vertex_project,
             vertex_location=vertex_location,
             vertex_auth_header=auth_header,
         )
+        rebuild, uncached = cache_rebuilds(transform, messages, optional_params)
+        request_body: Final = await build_with_accepted_thought_signatures_async(
+            transform, with_placeholder_thought_signatures, uncached
+        )
 
         ## LOGGING
         log_request: Final = partial(_log_generate_request, logging_obj, messages, api_base, headers)
@@ -3511,7 +3546,14 @@ class VertexLLM(VertexBase):
             async def post(body: RequestBody) -> ModelResponseIterator:
                 return await open_stream(client=client, data=json.dumps(body), request_body=body)
 
-            return (await post_with_fresh_cache_async(request_body, rebuild, post, log_request))[1]
+            accepted: Final = partial(
+                post_with_accepted_thought_signatures_async,
+                placeholders=with_placeholder_thought_signatures,
+                uncached=uncached,
+                post=post,
+                log_request=log_request,
+            )
+            return (await post_with_fresh_cache_async(request_body, rebuild, accepted, log_request))[1][1]
 
         streaming_response: Final = CustomStreamWrapper(
             completion_stream=None,
@@ -3578,23 +3620,17 @@ class VertexLLM(VertexBase):
             litellm_params=litellm_params,
         )
 
-        rebuild: Final = cache_rebuild(
-            partial(
-                async_transform_request_body,
-                **data,
-                vertex_project=vertex_project,
-                vertex_location=vertex_location,
-                vertex_auth_header=auth_header,
-            ),
-            messages,
-            optional_params,
-        )
-        request_body: Final = await async_transform_request_body(
+        transform: Final = partial(
+            async_transform_request_body,
             **data,
             vertex_project=vertex_project,
             vertex_location=vertex_location,
             vertex_auth_header=auth_header,
         )
+        rebuild, uncached = cache_rebuilds(transform, messages, optional_params)
+        request_body: Final = await build_with_accepted_thought_signatures_async(
+            transform, with_placeholder_thought_signatures, uncached
+        )
 
         _async_client_params: Final = {}
         if timeout:
@@ -3635,7 +3671,14 @@ class VertexLLM(VertexBase):
                 )
             return response
 
-        sent_body, first_response = await post_with_fresh_cache_async(request_body, rebuild, post, log_request)
+        accepted: Final = partial(
+            post_with_accepted_thought_signatures_async,
+            placeholders=with_placeholder_thought_signatures,
+            uncached=uncached,
+            post=post,
+            log_request=log_request,
+        )
+        _, (sent_body, first_response) = await post_with_fresh_cache_async(request_body, rebuild, accepted, log_request)
         return _transform_long_decode_hops(
             model=model,
             hops=await collect_hops_async(parse_body(first_response), sent_body, post),
@@ -3767,23 +3810,17 @@ class VertexLLM(VertexBase):
         )
 
         ## TRANSFORMATION ##
-        rebuild: Final = cache_rebuild(
-            partial(
-                sync_transform_request_body,
-                **transform_request_params,
-                vertex_project=vertex_project,
-                vertex_location=vertex_location,
-                vertex_auth_header=auth_header,
-            ),
-            messages,
-            optional_params,
-        )
-        data: Final = sync_transform_request_body(
+        transform: Final = partial(
+            sync_transform_request_body,
             **transform_request_params,
             vertex_project=vertex_project,
             vertex_location=vertex_location,
             vertex_auth_header=auth_header,
         )
+        rebuild, uncached = cache_rebuilds(transform, messages, optional_params)
+        data: Final = build_with_accepted_thought_signatures_sync(
+            transform, with_placeholder_thought_signatures, uncached
+        )
 
         ## LOGGING
         log_request: Final = partial(_log_generate_request, logging_obj, messages, url, headers)
@@ -3811,7 +3848,14 @@ class VertexLLM(VertexBase):
                 def post(body: RequestBody) -> ModelResponseIterator:
                     return open_stream(client=client, data=json.dumps(body), request_body=body)
 
-                return post_with_fresh_cache_sync(data, rebuild, post, log_request)[1]
+                accepted: Final = partial(
+                    post_with_accepted_thought_signatures_sync,
+                    placeholders=with_placeholder_thought_signatures,
+                    uncached=uncached,
+                    post=post,
+                    log_request=log_request,
+                )
+                return post_with_fresh_cache_sync(data, rebuild, accepted, log_request)[1][1]
 
             streaming_response: Final = CustomStreamWrapper(
                 completion_stream=None,
@@ -3855,7 +3899,14 @@ class VertexLLM(VertexBase):
                 )
             return response
 
-        sent_body, first_response = post_with_fresh_cache_sync(data, rebuild, post, log_request)
+        accepted: Final = partial(
+            post_with_accepted_thought_signatures_sync,
+            placeholders=with_placeholder_thought_signatures,
+            uncached=uncached,
+            post=post,
+            log_request=log_request,
+        )
+        _, (sent_body, first_response) = post_with_fresh_cache_sync(data, rebuild, accepted, log_request)
         return _transform_long_decode_hops(
             model=model,
             hops=collect_hops_sync(parse_body(first_response), sent_body, post),

litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py

@@ -9,12 +9,14 @@ their respective publisher-specific count-tokens endpoints.
 from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
-from pydantic import TypeAdapter
+from pydantic import JsonValue, TypeAdapter
 
+from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, filter_and_transform_beta_headers
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.anthropic.common_utils import (
     OBJECT_LIST_ADAPTER,
+    context_management_betas,
     count_tokens_feature_betas,
     is_claude_model_target,
     tools_without_custom_field,
@@ -31,6 +33,7 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
 
 
 _REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
+_CONTEXT_MANAGEMENT: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
 
 
 def count_tokens_request_body(
@@ -55,8 +58,14 @@ def count_tokens_publisher(model: str) -> str | None:
 
 def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
     authorization: Final = {"Authorization": f"Bearer {access_token}"}
-    betas: Final = count_tokens_feature_betas(
-        "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
+    betas: Final = (
+        *count_tokens_feature_betas(
+            "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
+        ),
+        *filter_and_transform_beta_headers(
+            list(context_management_betas(_CONTEXT_MANAGEMENT.validate_python(request_data.get("context_management")))),
+            "vertex_ai",
+        ),
     )
     return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization
 
@@ -193,6 +202,8 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
             api_base=litellm_params.get("api_base"),
         )
 
+        if request_data.get("context_management") is not None:
+            await aload_beta_headers_config()
         headers: Final = dict(count_tokens_headers(access_token, request_data))
 
         # Get async HTTP client

litellm/llms/vertex_ai/vertex_ai_partner_models/main.py

@@ -268,6 +268,7 @@ class VertexAIPartnerModels(VertexBase):
         system: object | None = None,
         tools: list[dict[str, object]] | None = None,
         count_options: Mapping[str, object] = MappingProxyType({}),
+        context_management: object = None,
     ):
         """
         Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.)
@@ -302,7 +303,15 @@ class VertexAIPartnerModels(VertexBase):
                 "model": model,
                 "messages": messages,
                 **MappingProxyType(
-                    {key: value for key, value in (("system", system), ("tools", tools)) if value is not None}
+                    {
+                        key: value
+                        for key, value in (
+                            ("system", system),
+                            ("tools", tools),
+                            ("context_management", context_management),
+                        )
+                        if value is not None
+                    }
                 ),
                 **count_options,
             }

litellm/messages/dispatch.py

@@ -75,9 +75,7 @@ def _public_request(
     return native_call(legacy, args, _native_kwargs(fields, kwargs))
 
 
-def _resolved(request: NativeCall) -> tuple[str, str | None]:
-    model: Final = str(request.resolved["model"])
-    provider: Final = optional_str(request.resolved.get("custom_llm_provider"))
+def _resolved_model(model: str, provider: str | None) -> tuple[str, str | None]:
     try:
         resolved_model, resolved_provider, _, _ = get_llm_provider(model, provider)
     except BadRequestError:
@@ -85,21 +83,43 @@ def _resolved(request: NativeCall) -> tuple[str, str | None]:
     return resolved_model, resolved_provider
 
 
-def _target(request: NativeCall, model: str, provider: str | None) -> Target:
-    requested_model: Final = str(request.resolved["model"])
-    model_info: Final = request.resolved.get("model_info")
-    api_base: Final = request.resolved.get("api_base")
-    native_config: Final = main._native_messages_config(  # pyright: ignore[reportPrivateUsage]  # the handler's own passthrough lookup
-        model, provider, model_info, api_base if isinstance(api_base, str) else None
-    )
+def _resolved(request: NativeCall) -> tuple[str, str | None]:
+    return _resolved_model(str(request.resolved["model"]), optional_str(request.resolved.get("custom_llm_provider")))
+
+
+def _route_target(
+    requested_model: str,
+    model: str,
+    provider: str | None,
+    model_info: object,
+    api_base: str | None,
+    compacting: bool,
+) -> Target:
+    native_config: Final = main._native_messages_config(model, provider, model_info, api_base)  # pyright: ignore[reportPrivateUsage]  # the handler's own passthrough lookup
     if native_config is not None:
         return Target.MESSAGES
-    compacting: Final = request.resolved.get("compaction") is not None
     if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info):  # pyright: ignore[reportPrivateUsage]  # the handler's own translation choice
         return Target.RESPONSES
     return Target.CHAT_COMPLETIONS
 
 
+def _target(request: NativeCall, model: str, provider: str | None) -> Target:
+    api_base: Final = request.resolved.get("api_base")
+    return _route_target(
+        str(request.resolved["model"]),
+        model,
+        provider,
+        request.resolved.get("model_info"),
+        api_base if isinstance(api_base, str) else None,
+        compacting=request.resolved.get("compaction") is not None,
+    )
+
+
+def messages_target(model: str, custom_llm_provider: str | None, model_info: object, api_base: str | None) -> Target:
+    resolved_model, resolved_provider = _resolved_model(model, custom_llm_provider)
+    return _route_target(model, resolved_model, resolved_provider, model_info, api_base, compacting=False)
+
+
 _FEATURES: Final[detectors.Detectors] = MappingProxyType(
     {
         **detectors.COMMON,
@@ -119,6 +139,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType(
     {
         **_FEATURES,
         Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook,
+        Feature.FOUNDRY_BETA_ROW: route_detectors.foundry_beta_row,
         Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception,
         Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted,
         Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry,

litellm/model_prices_and_context_window_backup.json

@@ -13478,6 +13478,10 @@
         "input_cost_per_token": 2e-07,
         "output_cost_per_token": 5e-07,
         "litellm_provider": "azure_ai",
+        "token_count_framing": 149,
+        "token_count_tool_framing": 302,
+        "token_count_responses_framing": 145,
+        "token_count_responses_tool_framing": 100,
         "max_input_tokens": 128000,
         "max_output_tokens": 128000,
         "mode": "chat",
@@ -13831,7 +13835,7 @@
         "input_cost_per_token_flex": 3.7e-07,
         "input_cost_per_token_priority": 1.295e-06,
         "litellm_provider": "bedrock",
-        "max_input_tokens": 163840,
+        "max_input_tokens": 156000,
         "max_output_tokens": 8000,
         "mode": "chat",
         "output_cost_per_token": 2.22e-06,
@@ -13845,7 +13849,7 @@
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false,
-        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
     },
     "bedrock/ap-northeast-1/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -14002,7 +14006,7 @@
         "input_cost_per_token_flex": 3.7e-07,
         "input_cost_per_token_priority": 1.295e-06,
         "litellm_provider": "bedrock",
-        "max_input_tokens": 163840,
+        "max_input_tokens": 156000,
         "max_output_tokens": 8000,
         "mode": "chat",
         "output_cost_per_token": 2.22e-06,
@@ -14016,7 +14020,7 @@
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false,
-        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
     },
     "bedrock/ap-south-1/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -14140,12 +14144,12 @@
         "input_cost_per_token": 7.4e-07,
         "input_cost_per_token_flex": 3.7e-07,
         "litellm_provider": "bedrock",
-        "max_input_tokens": 163840,
+        "max_input_tokens": 156000,
         "max_output_tokens": 8000,
         "mode": "chat",
         "output_cost_per_token": 2.22e-06,
         "output_cost_per_token_flex": 1.11e-06,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
         "supports_audio_input": false,
         "supports_function_calling": true,
         "supports_native_structured_output": true,
@@ -14250,7 +14254,7 @@
         "input_cost_per_token_flex": 3.7e-07,
         "input_cost_per_token_priority": 1.295e-06,
         "litellm_provider": "bedrock",
-        "max_input_tokens": 163840,
+        "max_input_tokens": 156000,
         "max_output_tokens": 8000,
         "mode": "chat",
         "output_cost_per_token": 2.22e-06,
@@ -14264,7 +14268,7 @@
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false,
-        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
     },
     "bedrock/eu-north-1/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -14786,7 +14790,7 @@
         "input_cost_per_token_flex": 3.7e-07,
         "input_cost_per_token_priority": 1.295e-06,
         "litellm_provider": "bedrock",
-        "max_input_tokens": 163840,
+        "max_input_tokens": 156000,
         "max_output_tokens": 8000,
         "mode": "chat",
         "output_cost_per_token": 2.22e-06,
@@ -14800,7 +14804,7 @@
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false,
-        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
     },
     "bedrock/sa-east-1/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -15035,7 +15039,7 @@
         "input_cost_per_token_flex": 3.1e-07,
         "input_cost_per_token_priority": 1.085e-06,
         "litellm_provider": "bedrock",
-        "max_input_tokens": 163840,
+        "max_input_tokens": 156000,
         "max_output_tokens": 8000,
         "mode": "chat",
         "output_cost_per_token": 1.85e-06,
@@ -15049,7 +15053,7 @@
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false,
-        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
     },
     "bedrock/us-east-1/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -15156,7 +15160,7 @@
         "input_cost_per_token_flex": 3.1e-07,
         "input_cost_per_token_priority": 1.085e-06,
         "litellm_provider": "bedrock",
-        "max_input_tokens": 163840,
+        "max_input_tokens": 156000,
         "max_output_tokens": 8000,
         "mode": "chat",
         "output_cost_per_token": 1.85e-06,
@@ -15170,7 +15174,7 @@
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false,
-        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
     },
     "bedrock/us-east-2/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -15552,6 +15556,7 @@
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-4-5.md"
     },
     "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
@@ -15562,6 +15567,7 @@
         "supports_pdf_input": true
     },
     "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
+        "deprecation_date": "2027-03-30",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
@@ -15708,7 +15714,7 @@
         "input_cost_per_token_flex": 3.1e-07,
         "input_cost_per_token_priority": 1.085e-06,
         "litellm_provider": "bedrock",
-        "max_input_tokens": 163840,
+        "max_input_tokens": 156000,
         "max_output_tokens": 8000,
         "mode": "chat",
         "output_cost_per_token": 1.85e-06,
@@ -15722,7 +15728,7 @@
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false,
-        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md"
     },
     "bedrock/us-west-2/minimax.minimax-m2.1": {
         "deprecation_date": "2027-03-30",
@@ -23519,12 +23525,12 @@
         ]
     },
     "deepgram/streaming/nova-3": {
-        "input_cost_per_second": 8e-05,
+        "input_cost_per_second": 0.00012833,
         "litellm_provider": "deepgram",
         "metadata": {
-            "calculation": "$0.0048/60 seconds = $0.00008000 per second",
+            "calculation": "$0.0077/60 seconds = $0.00012833 per second",
             "note": "Nova-3 monolingual streaming, pay as you go",
-            "original_pricing_per_minute": 0.0048
+            "original_pricing_per_minute": 0.0077
         },
         "mode": "audio_transcription",
         "output_cost_per_second": 0.0,
@@ -23533,13 +23539,28 @@
             "/v1/listen"
         ]
     },
-    "deepgram/streaming/nova-3-multilingual": {
-        "input_cost_per_second": 9.667e-05,
+    "deepgram/streaming/nova-3-medical": {
+        "input_cost_per_second": 0.00012833,
         "litellm_provider": "deepgram",
         "metadata": {
-            "calculation": "$0.0058/60 seconds = $0.00009667 per second",
+            "calculation": "$0.0077/60 seconds = $0.00012833 per second",
+            "note": "Nova-3 Medical streaming, launch price from Deepgram's 2025-03-03 announcement. deepgram.com/pricing does not list Nova-3 Medical, and this matches its current Nova-3 monolingual streaming pay as you go rate",
+            "original_pricing_per_minute": 0.0077
+        },
+        "mode": "audio_transcription",
+        "output_cost_per_second": 0.0,
+        "source": "https://www.businesswire.com/news/home/20250303470439/en",
+        "supported_endpoints": [
+            "/v1/listen"
+        ]
+    },
+    "deepgram/streaming/nova-3-multilingual": {
+        "input_cost_per_second": 0.00015333,
+        "litellm_provider": "deepgram",
+        "metadata": {
+            "calculation": "$0.0092/60 seconds = $0.00015333 per second",
             "note": "Nova-3 multilingual (language=multi) streaming, pay as you go",
-            "original_pricing_per_minute": 0.0058
+            "original_pricing_per_minute": 0.0092
         },
         "mode": "audio_transcription",
         "output_cost_per_second": 0.0,
@@ -24519,7 +24540,7 @@
         "input_cost_per_token_flex": 3.1e-07,
         "input_cost_per_token_priority": 1.085e-06,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 164000,
+        "max_input_tokens": 156000,
         "max_output_tokens": 8000,
         "mode": "chat",
         "output_cost_per_token": 1.85e-06,
@@ -24530,7 +24551,7 @@
         "supports_reasoning": true,
         "supports_native_structured_output": true,
         "supports_tool_choice": true,
-        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json",
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.md",
         "supports_audio_input": false,
         "supports_response_schema": true,
         "supports_vision": false
@@ -49614,6 +49635,7 @@
         "supports_max_reasoning_effort": true,
         "supports_mid_conversation_system": true,
         "supports_native_structured_output": false,
+        "supports_bedrock_native_structured_output": false,
         "supports_output_config": true,
         "supports_parallel_tool_use_config": true,
         "supports_pdf_input": true,
@@ -49658,6 +49680,7 @@
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_native_structured_output": false,
+        "supports_bedrock_native_structured_output": false,
         "supports_max_reasoning_effort": true,
         "supports_output_config": true,
         "bedrock_output_config_effort_ceiling": "xhigh",
@@ -53957,23 +53980,23 @@
         "supports_vision": true
     },
     "wandb/meta-llama/Llama-3.1-8B-Instruct": {
-        "max_input_tokens": 131000,
+        "max_input_tokens": 128000,
         "max_output_tokens": 128000,
         "input_cost_per_token": 2.2e-07,
         "output_cost_per_token": 2.2e-07,
         "litellm_provider": "wandb",
         "mode": "chat",
-        "source": "https://wandb.ai/site/pricing/tokens/"
+        "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference"
     },
     "wandb/deepseek-ai/DeepSeek-V3.1": {
         "supports_reasoning": true,
-        "max_input_tokens": 161000,
+        "max_input_tokens": 128000,
         "max_output_tokens": 128000,
         "input_cost_per_token": 5.5e-07,
         "output_cost_per_token": 1.65e-06,
         "litellm_provider": "wandb",
         "mode": "chat",
-        "source": "https://wandb.ai/site/pricing/tokens/"
+        "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference"
     },
     "wandb/meta-llama/Llama-3.3-70B-Instruct": {
         "max_input_tokens": 128000,
@@ -54457,6 +54480,8 @@
         "input_cost_per_token_above_200k_tokens_batches": 2e-06,
         "input_cost_per_token_batches": 1e-06,
         "litellm_provider": "xai",
+        "token_count_framing": 185,
+        "token_count_tool_framing": 214,
         "max_input_tokens": 1000000,
         "max_output_tokens": 1000000,
         "mode": "chat",
@@ -59875,6 +59900,7 @@
         "max_input_tokens": 922000,
         "max_output_tokens": 128000,
         "mode": "responses",
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-cyber.html",
         "use_openai_responses_path": true,
         "supported_endpoints": [
@@ -59896,7 +59922,8 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "supports_web_search": true
     },
     "bedrock_mantle/openai.gpt-daybreak-blue-5.6-sol": {
         "input_cost_per_token": 4.4e-06,
@@ -59911,6 +59938,7 @@
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "mode": "responses",
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "use_openai_responses_path": true,
         "supported_endpoints": [
             "/v1/responses"
@@ -59930,7 +59958,8 @@
         "supports_tool_choice": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-daybreak-blue-56-sol.html"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-daybreak-blue-56-sol.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/openai.gpt-5.6-luna": {
         "input_cost_per_token": 2.2e-07,
@@ -60336,6 +60365,7 @@
         "mode": "responses",
         "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495,
         "output_cost_per_token_ultrafast": 0.00033,
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "use_openai_responses_path": true,
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -60359,7 +60389,8 @@
         "supports_tool_choice": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/openai.gpt-6-sol": {
         "input_cost_per_token": 2.2e-06,
@@ -60374,6 +60405,7 @@
         "max_input_tokens": 922000,
         "max_output_tokens": 128000,
         "mode": "responses",
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "use_openai_responses_path": true,
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -60397,7 +60429,8 @@
         "supports_tool_choice": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/openai.gpt-6-luna": {
         "input_cost_per_token": 1.1e-07,
@@ -60412,6 +60445,7 @@
         "max_input_tokens": 922000,
         "max_output_tokens": 128000,
         "mode": "responses",
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "use_openai_responses_path": true,
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -60435,7 +60469,8 @@
         "supports_tool_choice": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html",
+        "supports_web_search": true
     },
     "us.openai.gpt-6-astra": {
         "supports_bedrock_runtime_chat_completions_response_format": true,
@@ -62630,7 +62665,9 @@
         "supports_response_schema": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "token_count_framing": 23,
+        "token_count_tool_framing": 12
     },
     "deepseek/deepseek-v4-flash-vision-exp": {
         "cache_creation_input_token_cost": 0.0,
@@ -62736,7 +62773,9 @@
         "supports_response_schema": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": false,
+        "token_count_framing": 76,
+        "token_count_tool_framing": 75
     },
     "tencent/deepseek-v4-pro": {
         "cache_creation_input_token_cost": 0.0,
@@ -64022,6 +64061,7 @@
         "input_cost_per_token": 3e-07,
         "input_cost_per_token_priority": 3.75e-07,
         "litellm_provider": "fireworks_ai",
+        "token_count_framing": 23,
         "max_input_tokens": 1048576,
         "max_output_tokens": 393216,
         "mode": "chat",
@@ -64029,6 +64069,7 @@
         "output_cost_per_token_priority": 1.5e-06,
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -64048,6 +64089,7 @@
         "output_cost_per_token_priority": 2.25e-06,
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -64090,6 +64132,7 @@
         ],
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -64120,6 +64163,7 @@
         "input_cost_per_token": 3e-07,
         "input_cost_per_token_priority": 3.75e-07,
         "litellm_provider": "fireworks_ai",
+        "token_count_framing": 23,
         "max_input_tokens": 1048576,
         "max_output_tokens": 393216,
         "mode": "chat",
@@ -64127,6 +64171,7 @@
         "output_cost_per_token_priority": 1.5e-06,
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -64146,6 +64191,7 @@
         "output_cost_per_token_priority": 2.25e-06,
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -64220,6 +64266,7 @@
         ],
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -64240,6 +64287,7 @@
         ],
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -64263,6 +64311,7 @@
         ],
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -64338,6 +64387,7 @@
         "output_cost_per_token_priority": 3e-06,
         "source": "https://api.fireworks.ai/v1/serverless/models?format=nested",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -64374,6 +64424,7 @@
         "output_cost_per_token_priority": 6.375e-06,
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_vision": true
     },
@@ -64410,6 +64461,7 @@
         "output_cost_per_token_priority": 3e-06,
         "source": "https://api.fireworks.ai/v1/serverless/models?format=nested",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -64480,6 +64532,7 @@
         ],
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -64503,6 +64556,7 @@
         ],
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -67296,6 +67350,7 @@
         "output_cost_per_token_priority": 5.5e-06,
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -67314,6 +67369,7 @@
         "output_cost_per_token_priority": 8.25e-06,
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -67332,6 +67388,7 @@
         "output_cost_per_token_priority": 5.5e-06,
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -67350,6 +67407,7 @@
         "output_cost_per_token_priority": 8.25e-06,
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -67365,6 +67423,7 @@
         "output_cost_per_token": 6.6e-06,
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -67407,6 +67466,7 @@
         "output_cost_per_token": 6.6e-06,
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -67425,6 +67485,7 @@
         "output_cost_per_token_priority": 6.25e-07,
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true
@@ -67443,6 +67504,7 @@
         "output_cost_per_token_priority": 9.375e-07,
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true
@@ -67460,6 +67522,7 @@
         "output_cost_per_token_priority": 6.25e-07,
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true
@@ -67478,6 +67541,7 @@
         "output_cost_per_token_priority": 9.375e-07,
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true
@@ -67496,6 +67560,7 @@
         "output_cost_per_token_priority": 4.05e-06,
         "source": "https://api.fireworks.ai/v1/serverless/models?format=nested",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true
@@ -68281,6 +68346,7 @@
         "supports_max_reasoning_effort": true,
         "supports_mid_conversation_system": true,
         "supports_native_structured_output": false,
+        "supports_bedrock_native_structured_output": false,
         "supports_output_config": true,
         "supports_parallel_tool_use_config": true,
         "supports_pdf_input": true,
@@ -68326,6 +68392,7 @@
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_native_structured_output": false,
+        "supports_bedrock_native_structured_output": false,
         "supports_max_reasoning_effort": true,
         "supports_output_config": true,
         "bedrock_output_config_effort_ceiling": "xhigh",
@@ -68554,6 +68621,7 @@
         "supports_max_reasoning_effort": true,
         "supports_mid_conversation_system": true,
         "supports_native_structured_output": false,
+        "supports_bedrock_native_structured_output": false,
         "supports_output_config": true,
         "supports_parallel_tool_use_config": true,
         "supports_pdf_input": true,
@@ -68594,6 +68662,7 @@
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_native_structured_output": false,
+        "supports_bedrock_native_structured_output": false,
         "supports_max_reasoning_effort": true,
         "supports_output_config": true,
         "bedrock_output_config_effort_ceiling": "xhigh",
@@ -68633,7 +68702,9 @@
         "cache_read_input_token_cost_above_272k_tokens": 5.4e-07,
         "output_cost_per_token": 1.62e-05,
         "output_cost_per_token_above_272k_tokens": 2.43e-05,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html"
+        "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144},
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": {
         "litellm_provider": "bedrock_mantle",
@@ -68668,7 +68739,9 @@
         "cache_read_input_token_cost_above_272k_tokens": 5.4e-08,
         "output_cost_per_token": 1.62e-06,
         "output_cost_per_token_above_272k_tokens": 2.43e-06,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html"
+        "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144},
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": {
         "litellm_provider": "bedrock_mantle",
@@ -68697,7 +68770,9 @@
         "input_cost_per_token": 3.375e-06,
         "cache_read_input_token_cost": 3.375e-07,
         "output_cost_per_token": 2.025e-05,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html"
+        "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144},
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/us-gov-west-1/xai.grok-4.3": {
         "use_openai_responses_path": true,
@@ -73731,6 +73806,7 @@
         "output_cost_per_token_priority": 1.875e-05,
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -75312,6 +75388,7 @@
         "output_cost_per_token_above_272k_tokens": 1.65e-05,
         "output_cost_per_token_above_272k_tokens_ultrafast": 9.9e-05,
         "output_cost_per_token_ultrafast": 6.6e-05,
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html",
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -75334,6 +75411,7 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true,
+        "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
         "use_openai_responses_path": true
     },
@@ -76758,6 +76836,7 @@
         "supports_audio_input": true,
         "supports_audio_output": false,
         "supports_computer_use": true,
+        "supports_continuation": true,
         "supports_function_calling": true,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
@@ -76817,6 +76896,7 @@
         ],
         "supports_audio_input": true,
         "supports_computer_use": true,
+        "supports_continuation": true,
         "supports_function_calling": true,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
@@ -76874,6 +76954,7 @@
             "text"
         ],
         "supports_audio_input": true,
+        "supports_continuation": true,
         "supports_function_calling": true,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
@@ -77231,6 +77312,7 @@
         "output_cost_per_token_priority": 1.5e-06,
         "source": "https://api.fireworks.ai/v1/serverless/models?format=nested",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -77389,6 +77471,7 @@
         "output_cost_per_token_priority": 9e-07,
         "source": "https://platform.minimax.io/docs/guides/pricing",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -79022,6 +79105,7 @@
         "mode": "responses",
         "output_cost_per_token": 5.5e-06,
         "output_cost_per_token_above_272k_tokens": 8.25e-06,
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-terra.html",
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -79044,6 +79128,7 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true,
+        "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
         "use_openai_responses_path": true
     },
@@ -81164,6 +81249,7 @@
         ],
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -81180,6 +81266,7 @@
         "output_cost_per_token": 3.08e-06,
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -81196,6 +81283,7 @@
         "output_cost_per_token": 8.4e-07,
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
@@ -81212,6 +81300,7 @@
         "output_cost_per_token": 8.4e-07,
         "source": "https://api.fireworks.ai/v1/serverless/models",
         "supports_function_calling": true,
+        "supports_mid_conversation_system": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,

litellm/proxy/anthropic_endpoints/endpoints.py

@@ -13,8 +13,6 @@ from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm.anthropic_interface.exceptions import (
-    AnthropicErrorDetail,
-    AnthropicErrorResponse,
     AnthropicExceptionMapping,
 )
 from litellm.integrations.custom_guardrail import ModifyResponseException
@@ -34,13 +32,14 @@ from litellm.proxy.common_request_processing import (
     resolve_counted_tool_fields,
     resolve_litellm_call_id,
 )
-from litellm.proxy.common_utils.error_body_call_id import error_body_call_id
+from litellm.proxy.common_utils.anthropic_error_payload import (
+    anthropic_handler_error_json_response,
+)
 from litellm.proxy.common_utils.http_parsing_utils import (  # noqa: F401  # legacy module exports
     _read_request_body,  # pyright: ignore[reportPrivateUsage,reportUnusedImport]  # backwards-compatible package export
     read_request_body,
 )
 from litellm.proxy.common_utils.openai_error_payload import (
-    LITELLM_CALL_ID_HEADER,
     error_status_code,
     litellm_call_id_headers,
     openai_error_param,
@@ -61,44 +60,8 @@ router: Final = APIRouter()
 _NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
 
 
-def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail:
-    if not exc.provider_specific_fields:
-        return detail
-    with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": exc.provider_specific_fields}
-    return with_fields
-
-
-def _anthropic_error_detail(
-    exc: ProxyException, detail: AnthropicErrorDetail, call_id: str | None
-) -> AnthropicErrorDetail:
-    if call_id is None:
-        return _with_provider_specific_fields(exc, detail)
-    with_call_id: Final[AnthropicErrorDetail] = {
-        **_with_provider_specific_fields(exc, detail),
-        "litellm_call_id": call_id,
-    }
-    return with_call_id
-
-
 def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse:
-    from litellm.proxy.proxy_server import (
-        _close_dangling_otel_server_span,  # pyright: ignore[reportPrivateUsage]  # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does
-        general_settings_view,
-    )
-
-    status_code: Final = int(exc.code) if exc.code is not None and exc.code.isdigit() else 500
-    _close_dangling_otel_server_span(request, status_code, exc=exc)
-    envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
-        status_code=status_code,
-        raw_message=exc.message,
-        request_id=request.headers.get("x-request-id"),
-    )
-    body_call_id: Final = error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER))
-    content: Final[AnthropicErrorResponse] = {
-        **envelope,
-        "error": _anthropic_error_detail(exc, envelope["error"], body_call_id),
-    }
-    return JSONResponse(status_code=status_code, content=content, headers=exc.headers)
+    return anthropic_handler_error_json_response(exc, request)
 
 
 _TOKEN_COUNT_FIELDS: Final = ("model", "messages", "tools", "system")

litellm/proxy/auth/auth_checks.py

@@ -125,6 +125,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
     NO_TEAM_MEMBERSHIP_SENTINEL,
     TAG_REGISTRY_OVERFLOW_SENTINEL,
     UserApiKeyCache,
+    budget_windows_absent_cache_key,
     end_user_cache_key,
     end_user_restricted_registry_cache_key,
     get_management_object_ttl,
@@ -3471,11 +3472,22 @@ async def delete_cache_key_object(
         )
 
     await publish_auth_cache_invalidation(cache_key=key)
+    await evict_budget_windows_absent_marker(
+        entity_type=Litellm_EntityType.KEY, entity_id=hashed_token, user_api_key_cache=user_api_key_cache
+    )
 
 
 _delete_cache_key_object: Final = delete_cache_key_object
 
 
+async def evict_budget_windows_absent_marker(
+    entity_type: Litellm_EntityType, entity_id: str, user_api_key_cache: UserApiKeyCache
+) -> None:
+    marker: Final = budget_windows_absent_cache_key(entity_type.value, entity_id)
+    evict_local(cache_keys=(marker,), user_api_key_cache=user_api_key_cache)
+    await publish_auth_cache_invalidation(cache_key=marker)
+
+
 async def delete_cache_key_objects(
     hashed_tokens: Sequence[str],
     user_api_key_cache: UserApiKeyCache,
@@ -5378,79 +5390,6 @@ async def _granted_model_lists(
     )
 
 
-async def _user_object_or_none(
-    valid_token: UserAPIKeyAuth,
-    prisma_client: PrismaClient,
-    user_api_key_cache: UserApiKeyCache,
-    proxy_logging_obj: ProxyLogging,
-) -> LiteLLM_UserTable | None:
-    try:
-        return await get_user_object(
-            user_id=valid_token.user_id,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            user_id_upsert=False,
-            proxy_logging_obj=proxy_logging_obj,
-        )
-    except UserNotFoundError:
-        return None
-
-
-async def enforced_model_allowlists(
-    valid_token: UserAPIKeyAuth,
-    prisma_client: PrismaClient | None,
-    user_api_key_cache: UserApiKeyCache,
-    proxy_logging_obj: ProxyLogging,
-) -> tuple[Sequence[str], ...]:
-    """One model allowlist per level that ``common_checks`` enforces on a request from this identity."""
-    key_models: Final = _resolve_key_models_for_auth_check(valid_token=valid_token)
-    if prisma_client is None:
-        return (key_models, tuple(valid_token.team_models or ()))
-    team_object: Final = (
-        None
-        if valid_token.team_id is None
-        else await get_team_object(
-            team_id=valid_token.team_id,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            proxy_logging_obj=proxy_logging_obj,
-        )
-    )
-    user_object: Final = (
-        None
-        if team_object is not None
-        else await _user_object_or_none(
-            valid_token=valid_token,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            proxy_logging_obj=proxy_logging_obj,
-        )
-    )
-    project_object: Final = (
-        None
-        if valid_token.project_id is None
-        else await get_project_object(
-            project_id=valid_token.project_id,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            proxy_logging_obj=proxy_logging_obj,
-        )
-    )
-    return (
-        key_models,
-        team_object.models if team_object is not None else (),
-        await _team_member_granted_models(
-            valid_token=valid_token,
-            team_object=team_object,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            proxy_logging_obj=proxy_logging_obj,
-        ),
-        user_object.models if user_object is not None else (),
-        project_object.models if project_object is not None else (),
-    )
-
-
 async def collect_matched_model_access_groups(
     model: str | Sequence[str] | None,
     valid_token: UserAPIKeyAuth | None,

litellm/proxy/client/cli/commands/agents.py

@@ -770,9 +770,10 @@ def codex_model_catalog(
     A model Codex does not know gets the fallback
     entry, with the same base instructions Codex itself uses so the agent never
     runs without a system prompt, and offers the reasoning efforts `groups` gives its model group
-    that the installed Codex, at `client_version`, can parse. Apart from aliases, the catalog is
-    the one `/v1/models?client_version=...` serves, built from the installed Codex's own entries
-    instead of the proxy's vendored ones.
+    that the installed Codex, at `client_version`, can parse. Apart from aliases, and the window the
+    deployments' `model_info.max_input_tokens` gives a model Codex knows, the catalog is the one
+    `/v1/models?client_version=...` serves, built from the installed Codex's own entries instead of
+    the proxy's vendored ones.
     """
     rows: Final = tuple(_codex_row(m, upstreams.get(m.id), groups) for m in models)
     body: Final = codex_models_response_json(

litellm/proxy/common_request_processing.py

@@ -30,6 +30,7 @@ import orjson
 from fastapi import HTTPException, Request, status
 from fastapi.responses import JSONResponse, Response, StreamingResponse
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
+from starlette.exceptions import HTTPException as StarletteHTTPException
 from starlette.types import Receive, Scope, Send
 from typing_extensions import Never
 
@@ -780,7 +781,7 @@ def resolve_counted_tool_fields(
     return resolved
 
 
-def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException:
+def proxy_exception_from_http_exception(exc: StarletteHTTPException, headers: dict[str, str]) -> ProxyException:
     raw_detail: Final = attribute_of(exc, "detail", str(exc))
     message, structured_fields = serialize_http_exception_detail(raw_detail)
     existing_fields: Final = getattr(exc, "provider_specific_fields", None) or {}

litellm/proxy/common_utils/anthropic_error_payload.py

@@ -0,0 +1,172 @@
+"""Auth, budget and model-access failures are raised by ``user_api_key_auth`` before the Messages
+handler runs, so the central handlers pick the Anthropic envelope from the request route."""
+
+from collections.abc import Callable, Mapping
+from types import MappingProxyType
+from typing import Final
+
+from fastapi import Request, status
+from fastapi.responses import JSONResponse
+from starlette.types import Scope
+
+from litellm.anthropic_interface.exceptions import (
+    ANTHROPIC_ERROR_TYPE_MAP,
+    AnthropicErrorDetail,
+    AnthropicErrorResponse,
+    AnthropicErrorType,
+    AnthropicExceptionMapping,
+)
+from litellm.proxy._types import ProxyException
+from litellm.proxy.auth.auth_utils import get_request_route
+from litellm.proxy.common_utils.error_body_call_id import error_body_call_id
+from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER, openai_error_type_for_status
+
+ANTHROPIC_MESSAGES_ROUTES: Final[frozenset[str]] = frozenset(
+    {
+        "/v1/messages",
+        "/v1/messages/count_tokens",
+        "/claude_code_gateway/v1/messages",
+        "/claude_code_gateway/v1/messages/count_tokens",
+    }
+)
+ANTHROPIC_PASS_THROUGH_PREFIX: Final = "/anthropic/"
+ANTHROPIC_SHAPED_MODEL_LIST_ROUTES: Final[frozenset[str]] = frozenset(
+    {"/v1/models", "/models", "/claude_code_gateway/v1/models"}
+)
+PROXY_ERROR_TYPE_BY_STATUS: Final[Mapping[int, AnthropicErrorType]] = MappingProxyType(
+    {**ANTHROPIC_ERROR_TYPE_MAP, 402: "billing_error", 409: "conflict_error", 504: "timeout_error"}
+)
+
+
+def proxy_error_type(status_code: int) -> AnthropicErrorType:
+    named: Final = PROXY_ERROR_TYPE_BY_STATUS.get(status_code)
+    if named is not None:
+        return named
+    if status_code == status.HTTP_408_REQUEST_TIMEOUT or not 400 <= status_code < 500:
+        return "api_error"
+    return "invalid_request_error"
+
+
+def uses_anthropic_error_envelope(request: Request) -> bool:
+    route: Final = get_request_route(request)
+    if route in ANTHROPIC_MESSAGES_ROUTES or route.startswith(ANTHROPIC_PASS_THROUGH_PREFIX):
+        return True
+    return route in ANTHROPIC_SHAPED_MODEL_LIST_ROUTES and "anthropic-version" in request.headers
+
+
+def route_error_content(
+    scope: Scope, anthropic_type: AnthropicErrorType, message: str, openai_content: Mapping[str, object]
+) -> Mapping[str, object]:
+    request: Final = Request(scope)
+    if not uses_anthropic_error_envelope(request):
+        return openai_content
+    content: Final[AnthropicErrorResponse] = {"type": "error", "error": {"type": anthropic_type, "message": message}}
+    request_id: Final = request.headers.get("x-request-id")
+    return content if request_id is None else {**content, "request_id": request_id}
+
+
+def anthropic_error_status_code(exc: ProxyException) -> int:
+    code: Final = exc.code
+    if code.isdigit():
+        return int(code)
+    return status.HTTP_500_INTERNAL_SERVER_ERROR
+
+
+def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail:
+    if not exc.provider_specific_fields:
+        return detail
+    with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": exc.provider_specific_fields}
+    return with_fields
+
+
+def _with_litellm_error_type(
+    exc: ProxyException, status_code: int, detail: AnthropicErrorDetail
+) -> AnthropicErrorDetail:
+    if not exc.type or exc.type in (detail["type"], openai_error_type_for_status(status_code)):
+        return detail
+    with_type: Final[AnthropicErrorDetail] = {**detail, "litellm_error_type": exc.type}
+    return with_type
+
+
+def _with_call_id(detail: AnthropicErrorDetail, call_id: str | None) -> AnthropicErrorDetail:
+    if call_id is None:
+        return detail
+    with_call_id: Final[AnthropicErrorDetail] = {**detail, "litellm_call_id": call_id}
+    return with_call_id
+
+
+def _anthropic_content(
+    exc: ProxyException,
+    status_code: int,
+    request_id: str | None,
+    general_settings: Mapping[str, object],
+    error_type: AnthropicErrorType | None,
+) -> AnthropicErrorResponse:
+    envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
+        status_code=status_code,
+        raw_message=exc.message,
+        request_id=request_id,
+        error_type=error_type,
+    )
+    body_call_id: Final = error_body_call_id(general_settings, exc.headers.get(LITELLM_CALL_ID_HEADER))
+    content: Final[AnthropicErrorResponse] = {
+        **envelope,
+        "error": _with_call_id(_with_provider_specific_fields(exc, envelope["error"]), body_call_id),
+    }
+    return content
+
+
+def anthropic_handler_error_content(
+    exc: ProxyException,
+    status_code: int,
+    request_id: str | None,
+    general_settings: Mapping[str, object],
+) -> AnthropicErrorResponse:
+    return _anthropic_content(exc, status_code, request_id, general_settings, None)
+
+
+def anthropic_error_content(
+    exc: ProxyException,
+    status_code: int,
+    request_id: str | None,
+    general_settings: Mapping[str, object],
+) -> AnthropicErrorResponse:
+    content: Final = _anthropic_content(exc, status_code, request_id, general_settings, proxy_error_type(status_code))
+    with_type: Final[AnthropicErrorResponse] = {
+        **content,
+        "error": _with_litellm_error_type(exc, status_code, content["error"]),
+    }
+    return with_type
+
+
+def _anthropic_json_response(
+    exc: ProxyException,
+    request: Request,
+    build_content: Callable[[ProxyException, int, str | None, Mapping[str, object]], AnthropicErrorResponse],
+) -> JSONResponse:
+    from litellm.proxy.proxy_server import (
+        _close_dangling_otel_server_span,  # pyright: ignore[reportPrivateUsage]  # proxy_server keeps the span-close helper private; a JSONResponse returned instead of a raised error must stamp the OTel server span like the global ProxyException handler does
+        general_settings_view,
+    )
+
+    status_code: Final = anthropic_error_status_code(exc)
+    _close_dangling_otel_server_span(request, status_code, exc=exc)
+    content: Final = build_content(exc, status_code, request.headers.get("x-request-id"), general_settings_view())
+    return JSONResponse(status_code=status_code, content=content, headers=exc.headers)
+
+
+def anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse:
+    return _anthropic_json_response(exc, request, anthropic_error_content)
+
+
+def anthropic_handler_error_json_response(exc: ProxyException, request: Request) -> JSONResponse:
+    return _anthropic_json_response(exc, request, anthropic_handler_error_content)
+
+
+def anthropic_internal_server_error_response(request: Request) -> JSONResponse:
+    content: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        raw_message="Internal server error",
+        request_id=request.headers.get("x-request-id"),
+    )
+    return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content=content)

litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py

@@ -183,7 +183,7 @@ def evict_local(cache_keys: Sequence[str], user_api_key_cache: "UserApiKeyCache"
 
 
 def _drop_from_memory(cache_key: str, user_api_key_cache: "UserApiKeyCache") -> None:
-    user_api_key_cache.in_memory_cache_for(cache_key).delete_cache(cache_key)
+    user_api_key_cache.delete_from_memory(cache_key)
 
 
 @with_service_target(AUTH_OBJECTS_TARGET)