Merge remote-tracking branch 'origin/litellm_internal_staging' into devin/1787857843-registry-audit-rolling

.github/workflows/sync-together-ai-models.yml

@@ -0,0 +1,68 @@
+name: Sync Together AI model registry
+
+on:
+  schedule:
+    - cron: "30 6 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  sync_together_ai_models:
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          ref: litellm_internal_staging
+          persist-credentials: false
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+      - name: Look for an already-open sync PR
+        id: existing
+        run: |
+          open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
+            --jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
+          echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
+          if [ -n "$open_pr" ]; then
+            echo "An open sync PR already exists on branch $open_pr; skipping this run."
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+      - name: Run the sync
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
+        env:
+          TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+      - name: Regenerate the JSON schema
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          uv run --frozen python ci_cd/generate_model_prices_schema.py
+      - name: Create a pull request when the registry changed
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          if git diff --quiet; then
+            echo "Registry already in sync; no PR needed."
+            exit 0
+          fi
+          branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$branch"
+          git add model_prices_and_context_window.json \
+            litellm/model_prices_and_context_window_backup.json \
+            model_prices_and_context_window.schema.json
+          git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
+          gh auth setup-git
+          git push origin "$branch"
+          gh pr create --title "feat(models): sync together_ai model registry" \
+            --body-file "$RUNNER_TEMP/pr_body.md" \
+            --head "$branch" \
+            --base litellm_internal_staging
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}

basedpyright-code-budget.json

@@ -6,10 +6,10 @@
     "limit": 2564
   },
   "reportAssignmentType": {
-    "limit": 320
+    "limit": 319
   },
   "reportAttributeAccessIssue": {
-    "limit": 483
+    "limit": 480
   },
   "reportCallIssue": {
     "limit": 113
@@ -30,7 +30,7 @@
     "limit": 7
   },
   "reportGeneralTypeIssues": {
-    "limit": 154
+    "limit": 105
   },
   "reportIncompatibleMethodOverride": {
     "limit": 56
@@ -99,19 +99,19 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44528
+    "limit": 44526
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38804
+    "limit": 38782
   },
   "reportUnknownParameterType": {
     "limit": 19829
   },
   "reportUnknownVariableType": {
-    "limit": 30355
+    "limit": 30349
   },
   "reportUnnecessaryCast": {
     "limit": 117
@@ -123,7 +123,7 @@
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 833
+    "limit": 831
   },
   "reportUntypedBaseClass": {
     "limit": 0

ci_cd/generate_model_prices_schema.py

@@ -73,6 +73,11 @@ ARRAY_KEYS: dict[str, JsonSchema] = {
         "description": "Output modalities the model can produce.",
         "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
     },
+    "reasoning_effort_levels": {
+        "type": "array",
+        "description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
+        "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
+    },
     "supported_regions": {
         "type": "array",
         "description": "Cloud regions the model is available in ('global' or region ids).",

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -12,7 +12,7 @@ Endpoints for /project operations
 
 import json
 from collections.abc import Sequence
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Final
 
 from fastapi import APIRouter, Depends, HTTPException, Request
 
@@ -35,6 +35,8 @@ if TYPE_CHECKING:
         LiteLLM_VerificationTokenActions,
     )
 
+    from litellm import Router
+
 router = APIRouter()
 
 
@@ -205,6 +207,114 @@ def _check_team_project_limits(
         )
 
 
+def _project_models_missing_positive_quota(
+    models: list[str] | None,
+    rpm_limits: Mapping[str, object] | None,
+    tpm_limits: Mapping[str, object] | None,
+) -> list[str]:
+    """Return the models that lack a positive `rpm` AND `tpm` quota.
+
+    A valid quota is a positive integer; null, zero, and negative are rejected
+    because downstream rate limiters treat a non-positive limit as immediately
+    exhausted (every request blocked).
+    """
+
+    def _is_positive(value: object) -> bool:
+        return isinstance(value, int) and not isinstance(value, bool) and value > 0
+
+    rpm = rpm_limits or {}
+    tpm = tpm_limits or {}
+    return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))]
+
+
+def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
+    return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
+
+
+def _project_models_expanding_at_request_time(
+    models: Sequence[str] | None, access_group_names: frozenset[str]
+) -> tuple[str, ...]:
+    """Entries project auth expands to many concrete models (`all-proxy-models`, `*` patterns,
+    access groups). The rate limiter looks quotas up by the exact requested model name, so a
+    quota keyed on one of these entries is never applied."""
+    return tuple(
+        model
+        for model in (models or ())
+        if model == SpecialModelNames.all_proxy_models.value or "*" in model or model in access_group_names
+    )
+
+
+def _raise_on_project_models_expanding_at_request_time(
+    models: Sequence[str] | None, access_group_names: frozenset[str]
+) -> None:
+    expanding: Final = _project_models_expanding_at_request_time(models, access_group_names)
+    if not expanding:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while 'enforce_project_model_quota' is enabled. List concrete model names instead."
+        },
+    )
+
+
+def _raise_on_missing_project_model_quota(
+    data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset()
+) -> None:
+    """Require a positive `rpm`/`tpm` quota for every model on project CREATE.
+
+    `model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request
+    model's `set_model_info` validator, so they are read from there.
+
+    Only invoked when `general_settings.enforce_project_model_quota` is enabled
+    (default off), so it is opt-in and does not change behavior for existing users.
+    """
+    _raise_on_project_models_expanding_at_request_time(data.models, access_group_names)
+    metadata = data.metadata or {}
+    missing = _project_models_missing_positive_quota(
+        data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit")
+    )
+    if not missing:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {missing} added to project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
+        },
+    )
+
+
+def _raise_on_missing_project_model_quota_on_update(
+    data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset()
+) -> None:
+    """Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE.
+
+    `/project/update` replaces `models` and `metadata` when they are provided, so the
+    check runs on what the project WILL look like: a partial update that doesn't touch
+    models/quota keeps the existing values, while one that adds a model or clears a
+    model's quota must leave every resulting model with a positive limit.
+
+    Only invoked when `general_settings.enforce_project_model_quota` is enabled
+    (default off), so it is opt-in and does not change behavior for existing users.
+    """
+    resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or [])
+    resulting_metadata = (
+        data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {})
+    )
+    _raise_on_project_models_expanding_at_request_time(resulting_models, access_group_names)
+    missing = _project_models_missing_positive_quota(
+        resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit")
+    )
+    if not missing:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {missing} would be left on the project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
+        },
+    )
+
+
 async def _create_budget_for_project(
     data: NewProjectRequest,
     user_id: str | None,
@@ -352,7 +462,9 @@ async def new_project(
     ```
     """
     from litellm.proxy.proxy_server import (
+        general_settings,
         litellm_proxy_admin_name,
+        llm_router,
         premium_user,
         prisma_client,
     )
@@ -399,6 +511,10 @@ async def new_project(
             data=data,
         )
 
+        # Opt-in (default off): require rpm/tpm for every model added to the project.
+        if general_settings.get("enforce_project_model_quota", False):
+            _raise_on_missing_project_model_quota(data, _router_access_group_names(llm_router))
+
         # Check if user has permission to create projects for this team
         # only team admins can create projects for their team
         has_permission = await _check_user_permission_for_project(
@@ -538,7 +654,9 @@ async def update_project(
     ```
     """
     from litellm.proxy.proxy_server import (
+        general_settings,
         litellm_proxy_admin_name,
+        llm_router,
         premium_user,
         prisma_client,
         user_api_key_cache,
@@ -642,6 +760,12 @@ async def update_project(
                 data=data,
             )
 
+        # Opt-in (default off): require rpm/tpm for every model the update would leave on the project.
+        if general_settings.get("enforce_project_model_quota", False):
+            _raise_on_missing_project_model_quota_on_update(
+                data, existing_project, _router_access_group_names(llm_router)
+            )
+
         # Prepare update data
         update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
         update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name

litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql

@@ -0,0 +1,18 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)

litellm/_logging.py

@@ -5,7 +5,7 @@ import os
 import sys
 from datetime import datetime
 from logging import Formatter
-from typing import Any, Final
+from typing import Any, Final, TextIO
 
 import litellm
 from litellm.constants import (
@@ -234,11 +234,65 @@ class CorrelationContextFilter(logging.Filter):
 _correlation_filter: Final = CorrelationContextFilter()
 
 
-json_logs = bool(os.getenv("JSON_LOGS", False))
+_LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s"
+_LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s"
+_PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX
+_COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}"
+
+
+def _stream_is_tty(stream: TextIO | None) -> bool:
+    """True when the stream is an open interactive terminal; never raises.
+
+    A stream can be None (pythonw/embedded interpreters), lack isatty entirely
+    (GUI log-redirect shims), or be closed; import must survive all three.
+    """
+    try:
+        return stream is not None and stream.isatty()
+    except (AttributeError, ValueError):
+        return False
+
+
+def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
+    """The plain-text log format, colorized only when both streams are an interactive terminal.
+
+    Honors the NO_COLOR convention from no-color.org: color is disabled when
+    NO_COLOR is present with a non-empty value.
+    """
+    if os.environ.get("NO_COLOR"):
+        return _PLAIN_LOG_FORMAT
+    return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT
+
+
+class LevelRoutingStreamHandler(logging.StreamHandler):
+    """Writes records below WARNING to stdout and WARNING and above to stderr.
+
+    Collectors that derive severity from the stream report every stderr line as an error.
+    """
+
+    def emit(self, record: logging.LogRecord) -> None:
+        preferred: Final = sys.stdout if record.levelno < logging.WARNING else sys.stderr
+        if preferred is None or getattr(preferred, "closed", False):
+            self.stream = sys.stderr  # rebind-ok: fall back to the pre-fix stream rather than raising per record
+        else:
+            self.stream = preferred  # rebind-ok: StreamHandler.emit writes self.stream under the handler lock
+        super().emit(record)
+
+
+def _parse_json_logs_env(value: str | None) -> bool:
+    """Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs.
+
+    Matches the reader in litellm-proxy-extras/_logging.py. The previous
+    bool(os.getenv(...)) treated any non-empty value, including "false" and "0",
+    as enabled.
+    """
+    return (value or "").lower() == "true"
+
+
+json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS"))
 # Create a handler for the logger (you may need to adapt this based on your needs)
 log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
 numeric_level: Final[str] = getattr(logging, log_level.upper())
-handler: Final = logging.StreamHandler()
+handler: Final = LevelRoutingStreamHandler()
 handler.setLevel(numeric_level)
 handler.addFilter(_secret_filter)
 handler.addFilter(_correlation_filter)
@@ -447,7 +501,7 @@ if json_logs:
     _setup_json_exception_handlers(JsonFormatter())
 else:
     formatter: Final = CorrelationPlainFormatter(
-        "\033[92m%(asctime)s - %(name)s:%(levelname)s\033[0m: %(filename)s:%(lineno)s - %(message)s",
+        _plain_log_format(sys.stdout, sys.stderr),
         datefmt="%H:%M:%S",
     )
 
@@ -628,7 +682,7 @@ def _turn_on_json():
 
     - Adds a JSON formatter to all loggers
     """
-    handler: Final = logging.StreamHandler()
+    handler: Final = LevelRoutingStreamHandler()
     handler.setFormatter(JsonFormatter())
     _initialize_loggers_with_handler(handler)
     # Set up exception handlers

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -59,9 +59,11 @@ if TYPE_CHECKING:
     from litellm.types.llms.openai import (
         ALL_RESPONSES_API_TOOL_PARAMS,
         AllMessageValues,
+        ChatCompletionFileObject,
         ChatCompletionImageObject,
         ChatCompletionRedactedThinkingBlock,
         ChatCompletionThinkingBlock,
+        ChatCompletionToolReferenceObject,
         OpenAIMessageContentListBlock,
     )
     from litellm.types.utils import Choices
@@ -175,6 +177,16 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li
     return "length"
 
 
+def _input_file_from_file_value(file_value: object) -> dict[str, object]:
+    if not isinstance(file_value, dict):
+        return {"type": "input_file"}
+    file_dict: Final = cast("dict[str, object]", file_value)  # cast-ok: runtime dict checked
+    return {
+        "type": "input_file",
+        **{key: file_dict[key] for key in ("file_id", "file_data", "filename") if key in file_dict},
+    }
+
+
 def _incomplete_reason_from_response_payload(response_payload: object) -> str | None:
     if not isinstance(response_payload, Mapping):
         return None
@@ -957,7 +969,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         content: str
         | list[object]
         | Iterable[
-            Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock", "ChatCompletionRedactedThinkingBlock"]
+            Union[
+                "OpenAIMessageContentListBlock",
+                "ChatCompletionThinkingBlock",
+                "ChatCompletionRedactedThinkingBlock",
+                "ChatCompletionToolReferenceObject",
+            ]
         ]
         | None,
         role: str,
@@ -1006,17 +1023,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   image -> %s", converted)
                         elif item_type == "file":
-                            # Map Chat Completion file to Responses API input_file
-                            # {"type": "file", "file": {"file_data": "...", "filename": "..."}}
-                            # -> {"type": "input_file", "file_data": "...", "filename": "..."}
-                            file_data = item.get("file", {})
-                            converted = {"type": "input_file"}
-                            if isinstance(file_data, dict):
-                                for key in ["file_id", "file_data", "filename"]:
-                                    if key in file_data:
-                                        converted[key] = file_data[key]
+                            converted = _input_file_from_file_value(
+                                cast("ChatCompletionFileObject", item).get("file"),  # cast-ok: type tag checked
+                            )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   file -> %s", converted)
+                        elif item_type == "tool_reference":
+                            verbose_logger.debug(
+                                "Chat provider:   tool_reference has no responses API equivalent; skipped"
+                            )
                         elif item_type in [
                             "input_text",
                             "input_image",

litellm/constants.py

@@ -296,6 +296,9 @@ GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS: Final = int(
     os.getenv("GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS", 24 * 60 * 60)
 )
 BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS: Final = 25_000
+DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES: Final = 500_000
+PRESIDIO_ANALYZE_CHUNK_OVERLAP_CHARS: Final = 4096
+PRESIDIO_ANALYZE_CHUNK_CONCURRENCY: Final = 8
 # Aggregation threshold: default to 80% of the asyncio queue maxsize so the check can always trigger.
 # Must be < LITELLM_ASYNCIO_QUEUE_MAXSIZE; if set higher the aggregation logic will never fire.
 MAX_SIZE_IN_MEMORY_QUEUE: Final = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8)))

litellm/cost_calculator.py

@@ -76,7 +76,10 @@ from litellm.llms.perplexity.cost_calculator import (
 from litellm.llms.tencent.cost_calculator import (
     cost_per_token as tencent_cost_per_token,
 )
-from litellm.llms.together_ai.cost_calculator import get_model_params_and_category
+from litellm.llms.together_ai.cost_calculator import (
+    get_model_params_and_category,
+    has_together_registry_pricing,
+)
 from litellm.llms.vertex_ai.cost_calculator import (
     cost_per_character as google_cost_per_character,
 )
@@ -1569,10 +1572,9 @@ def completion_cost(
 
                     return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj)
                 # Calculate cost based on prompt_tokens, completion_tokens
-                if "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai":
-                    # together ai prices based on size of llm
-                    # get_model_params_and_category takes a model name and returns the category of LLM size it is in model_prices_and_context_window.json
-
+                if (
+                    "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
+                ) and not has_together_registry_pricing(model, litellm.model_cost):
                     model = get_model_params_and_category(model, call_type=CallTypes(call_type))
 
                 # replicate llms are calculate based on time for request running

litellm/experimental_mcp_client/client.py

@@ -56,6 +56,9 @@ from litellm.types.mcp import (
     MCPStdioConfig,
     MCPTransport,
     MCPTransportType,
+    credential_redirect_hook,
+    has_header,
+    without_header,
 )
 
 
@@ -273,6 +276,7 @@ class MCPClient:
         transport_type: MCPTransportType = MCPTransport.http,
         auth_type: MCPAuthType = None,
         auth_value: str | dict[str, str] | None = None,
+        auth_header_name: str | None = None,
         timeout: float | None = None,
         stdio_config: MCPStdioConfig | None = None,
         extra_headers: dict[str, str] | None = None,
@@ -288,6 +292,11 @@ class MCPClient:
         self.auth_type: MCPAuthType = auth_type
         self.timeout: float = timeout if timeout is not None else MCP_CLIENT_TIMEOUT
         self._mcp_auth_value: str | dict[str, str] | None = None
+        # The one place this client decides which header its credential occupies: the operator's
+        # configured slot on the v1 path, or the slot the v2 resolver's auth object already owns.
+        # Every consumer reads this rather than re-deriving it, since each re-derivation so far
+        # picked up a different bug.
+        self._credential_slot: str | None = auth_header_name or getattr(resolved_auth, "header_name", None)
         self.stdio_config: MCPStdioConfig | None = stdio_config
         self.extra_headers: dict[str, str] | None = extra_headers
         self.ssl_verify: VerifyTypes | None = ssl_verify
@@ -501,26 +510,33 @@ class MCPClient:
         else:
             self._mcp_auth_value = mcp_auth_value
 
+    def _header_slot(self, default: str) -> str:
+        return self._credential_slot or default
+
     def _get_auth_headers(self) -> dict:
         """Generate authentication headers based on auth type."""
         headers: Final = {}
         if self._mcp_auth_value:
             if isinstance(self._mcp_auth_value, str):
                 if self.auth_type == MCPAuth.bearer_token:
-                    headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
+                    static_bearer: Final = strip_auth_scheme(self._mcp_auth_value, "Bearer")
+                    headers[self._header_slot("Authorization")] = f"Bearer {static_bearer}"
                 elif self.auth_type == MCPAuth.basic:
-                    headers["Authorization"] = f"Basic {self._mcp_auth_value}"
+                    headers[self._header_slot("Authorization")] = f"Basic {self._mcp_auth_value}"
                 elif self.auth_type == MCPAuth.api_key:
-                    headers["X-API-Key"] = self._mcp_auth_value
+                    headers[self._header_slot("X-API-Key")] = self._mcp_auth_value
                 elif self.auth_type == MCPAuth.authorization:
                     # This auth type means the caller owns the whole header value.
-                    headers["Authorization"] = self._mcp_auth_value
+                    headers[self._header_slot("Authorization")] = self._mcp_auth_value
                 elif self.auth_type == MCPAuth.oauth2:
-                    headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
+                    oauth2_bearer: Final = strip_auth_scheme(self._mcp_auth_value, "Bearer")
+                    headers[self._header_slot("Authorization")] = f"Bearer {oauth2_bearer}"
                 elif self.auth_type == MCPAuth.token:
-                    headers["Authorization"] = f"token {strip_auth_scheme(self._mcp_auth_value, 'token')}"
+                    scheme_token: Final = strip_auth_scheme(self._mcp_auth_value, "token")
+                    headers[self._header_slot("Authorization")] = f"token {scheme_token}"
                 elif self.auth_type == MCPAuth.oauth2_token_exchange:
-                    headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
+                    exchanged_bearer: Final = strip_auth_scheme(self._mcp_auth_value, "Bearer")
+                    headers[self._header_slot("Authorization")] = f"Bearer {exchanged_bearer}"
             elif isinstance(self._mcp_auth_value, dict):
                 headers.update(self._mcp_auth_value)
         # Note: aws_sigv4 auth is not handled here — SigV4 requires per-request
@@ -528,7 +544,14 @@ class MCPClient:
         # of static headers. See MCPSigV4Auth and _create_httpx_client_factory().
         # update the headers with the extra headers
         if self.extra_headers:
-            headers.update(self.extra_headers)
+            # Mirrors _resolve_v2_auth: when the operator named a slot for the credential the
+            # gateway resolved, no injected header may shadow it, case-insensitively, since HTTP
+            # header names are. Without a configured slot the old precedence stands unchanged.
+            slot: Final = self._credential_slot
+            injected: Final = (
+                without_header(self.extra_headers, slot) if slot and has_header(headers, slot) else self.extra_headers
+            )
+            headers.update(injected or {})
         return _strip_header_whitespace(headers)
 
     def _create_httpx_client_factory(self) -> Callable[..., httpx.AsyncClient]:
@@ -556,12 +579,14 @@ class MCPClient:
             # SigV4 aws_auth. Both are None for the common case — no behavior change.
             fallback_auth: Final = self._resolved_auth if self._resolved_auth is not None else self._aws_auth
             effective_auth: Final = auth if auth is not None else fallback_auth
+            guard: Final = credential_redirect_hook(self.server_url, self._credential_slot)
             return httpx.AsyncClient(
                 headers=headers,
                 timeout=timeout,
                 auth=effective_auth,
                 verify=ssl_config,
                 follow_redirects=True,
+                event_hooks={"request": [guard]} if guard else {},
             )
 
         return factory

litellm/integrations/SlackAlerting/batching_handler.py

@@ -10,6 +10,8 @@ from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_proxy_logger
 
+from .ms_teams import MS_TEAMS_ALERTING_DESTINATION, build_ms_teams_payload
+
 if TYPE_CHECKING:
     from .slack_alerting import SlackAlerting as _SlackAlerting
 
@@ -62,14 +64,17 @@ async def send_to_webhook(slackAlertingInstance: SlackAlertingType, item, count)
         if count > 1:
             payload["text"] = f"[Num Alerts: {count}]\n\n{payload['text']}"
 
+        request_body: Final = (
+            build_ms_teams_payload(payload["text"]) if item.get("format") == MS_TEAMS_ALERTING_DESTINATION else payload
+        )
         response: Final = await slackAlertingInstance.async_http_handler.post(
             url=item["url"],
             headers=item["headers"],
-            data=json.dumps(payload),
+            data=json.dumps(request_body),
         )
         if response.status_code != 200:
-            verbose_proxy_logger.debug("Error sending slack alert to url=%s. Error=%s", item["url"], response.text)
+            verbose_proxy_logger.debug("Error sending alert to url=%s. Error=%s", item["url"], response.text)
     except Exception as e:
-        verbose_proxy_logger.debug("Error sending slack alert: %s", e)
+        verbose_proxy_logger.debug("Error sending alert: %s", e)
     finally:
         _print_alerting_payload_warning(payload, slackAlertingInstance=slackAlertingInstance)

litellm/integrations/SlackAlerting/ms_teams.py

@@ -0,0 +1,75 @@
+"""Microsoft Teams alert delivery helpers.
+
+Teams incoming webhooks (Workflows and legacy connectors) accept an Adaptive
+Card wrapped in a message attachment, so alert text is delivered as a single
+wrapped TextBlock.
+"""
+
+import os
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm.types.integrations.slack_alerting import AlertType
+
+MS_TEAMS_WEBHOOK_URL_ENV: Final = "MS_TEAMS_WEBHOOK_URL"
+
+MS_TEAMS_ALERTING_DESTINATION: Final = "ms_teams"
+
+MS_TEAMS_ALERT_HEADERS: Final[Mapping[str, str]] = MappingProxyType({"Content-type": "application/json"})
+
+
+class MSTeamsTextBlock(TypedDict):
+    type: ReadOnly[str]
+    text: ReadOnly[str]
+    wrap: ReadOnly[bool]
+
+
+class MSTeamsAdaptiveCard(TypedDict):
+    type: ReadOnly[str]
+    version: ReadOnly[str]
+    body: ReadOnly[tuple[MSTeamsTextBlock, ...]]
+
+
+class MSTeamsAttachment(TypedDict):
+    contentType: ReadOnly[str]
+    content: ReadOnly[MSTeamsAdaptiveCard]
+
+
+class MSTeamsMessage(TypedDict):
+    type: ReadOnly[str]
+    attachments: ReadOnly[tuple[MSTeamsAttachment, ...]]
+
+
+class MSTeamsAlertText(TypedDict):
+    text: ReadOnly[str]
+
+
+class MSTeamsQueueItem(TypedDict):
+    url: ReadOnly[str]
+    headers: ReadOnly[Mapping[str, str]]
+    payload: ReadOnly[MSTeamsAlertText]
+    alert_type: ReadOnly[AlertType]
+    format: ReadOnly[str]
+
+
+def get_ms_teams_webhook_url() -> str | None:
+    return os.getenv(MS_TEAMS_WEBHOOK_URL_ENV)
+
+
+def build_ms_teams_payload(text: str) -> MSTeamsMessage:
+    return MSTeamsMessage(
+        type="message",
+        attachments=(
+            MSTeamsAttachment(
+                contentType="application/vnd.microsoft.card.adaptive",
+                content=MSTeamsAdaptiveCard(
+                    type="AdaptiveCard",
+                    version="1.4",
+                    body=(MSTeamsTextBlock(type="TextBlock", text=text, wrap=True),),
+                ),
+            ),
+        ),
+    )

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -57,6 +57,13 @@ from litellm.types.proxy.model_deprecation import (
 
 from ..email_templates.templates import *
 from .batching_handler import send_to_webhook, squash_payloads
+from .ms_teams import (
+    MS_TEAMS_ALERT_HEADERS,
+    MS_TEAMS_ALERTING_DESTINATION,
+    MSTeamsAlertText,
+    MSTeamsQueueItem,
+    get_ms_teams_webhook_url,
+)
 from .utils import process_slack_alerting_variables
 
 if TYPE_CHECKING:
@@ -1431,13 +1438,45 @@ Model Info:
             # only send budget alerts over Email
             await self.send_email_alert_using_smtp(webhook_event=user_info, alert_type=alert_type)
 
-        if "slack" not in self.alerting:
+        send_to_slack: Final = "slack" in self.alerting
+        send_to_ms_teams: Final = MS_TEAMS_ALERTING_DESTINATION in self.alerting
+        if not send_to_slack and not send_to_ms_teams:
             return
         if alert_type not in self.alert_types:
             return
 
         from datetime import datetime
 
+        # Get the current timestamp
+        current_time: Final = datetime.now().strftime("%H:%M:%S")
+        _proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None)
+        # Use .name if it's an enum, otherwise use as is
+        alert_type_name: Final = getattr(alert_type, "name", alert_type)
+        alert_type_formatted: Final = f"Alert type: `{alert_type_name}`"
+        if alert_type == "daily_reports" or alert_type == "new_model_added":
+            formatted_message = alert_type_formatted + message
+        else:
+            formatted_message = (
+                f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}"
+            )
+
+        if kwargs:
+            for key, value in kwargs.items():
+                formatted_message += f"\n\n{key}: `{value}`\n\n"
+        if alerting_metadata:
+            for key, value in alerting_metadata.items():
+                formatted_message += f"\n\n*Alerting Metadata*: \n{key}: `{value}`\n\n"
+        if _proxy_base_url is not None:
+            formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`"
+
+        if send_to_ms_teams:
+            self._enqueue_ms_teams_alert(formatted_message=formatted_message, alert_type=alert_type)
+
+        if not send_to_slack:
+            if len(self.log_queue) >= self.batch_size:
+                await self.flush_queue()
+            return
+
         # Check if digest mode is enabled for this alert type
         alert_type_name_str: Final = getattr(alert_type, "value", str(alert_type))
         _atc: Final = self.alert_type_config.get(alert_type_name_str)
@@ -1473,28 +1512,6 @@ Model Info:
                     )
             return  # Suppress immediate alert; will be emitted by _flush_digest_buckets
 
-        # Get the current timestamp
-        current_time: Final = datetime.now().strftime("%H:%M:%S")
-        _proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None)
-        # Use .name if it's an enum, otherwise use as is
-        alert_type_name: Final = getattr(alert_type, "name", alert_type)
-        alert_type_formatted: Final = f"Alert type: `{alert_type_name}`"
-        if alert_type == "daily_reports" or alert_type == "new_model_added":
-            formatted_message = alert_type_formatted + message
-        else:
-            formatted_message = (
-                f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}"
-            )
-
-        if kwargs:
-            for key, value in kwargs.items():
-                formatted_message += f"\n\n{key}: `{value}`\n\n"
-        if alerting_metadata:
-            for key, value in alerting_metadata.items():
-                formatted_message += f"\n\n*Alerting Metadata*: \n{key}: `{value}`\n\n"
-        if _proxy_base_url is not None:
-            formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`"
-
         # check if we find the slack webhook url in self.alert_to_webhook_url
         if self.alert_to_webhook_url is not None and alert_type in self.alert_to_webhook_url:
             slack_webhook_url: str | list[str] | None = self.alert_to_webhook_url[alert_type]
@@ -1531,6 +1548,24 @@ Model Info:
         if len(self.log_queue) >= self.batch_size:
             await self.flush_queue()
 
+    def _enqueue_ms_teams_alert(self, formatted_message: str, alert_type: AlertType) -> None:
+        ms_teams_webhook_url: Final = get_ms_teams_webhook_url()
+        if ms_teams_webhook_url is None:
+            verbose_proxy_logger.error(
+                "MS Teams alerting is enabled but MS_TEAMS_WEBHOOK_URL is not set. Dropping alert type=%s",
+                alert_type,
+            )
+            return
+        payload: Final[MSTeamsAlertText] = {"text": formatted_message}
+        item: Final[MSTeamsQueueItem] = {
+            "url": ms_teams_webhook_url,
+            "headers": MS_TEAMS_ALERT_HEADERS,
+            "payload": payload,
+            "alert_type": alert_type,
+            "format": MS_TEAMS_ALERTING_DESTINATION,
+        }
+        self.log_queue.append(item)
+
     async def async_send_batch(self):
         if not self.log_queue:
             return

litellm/integrations/anthropic_cache_control_hook.py

@@ -24,6 +24,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     with_prompt_cache_breakpoint,
 )
 from litellm.types.integrations.anthropic_cache_control_hook import (
+    GATEWAY_INJECTED_CACHE_METADATA_KEY,
+    GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
     CacheControlInjectionPoint,
     CacheControlMessageInjectionPoint,
 )
@@ -185,7 +187,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         reserved_blocks: Final = (
             1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
         )
-        breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
+        breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
         processed_messages = self._apply_message_injections(
             points=applied_message_points,
             messages=processed_messages,
@@ -194,7 +196,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         if (
             openai_dialect
-            and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before
+            and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before
         ):
             non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
 
@@ -236,7 +238,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         return provider
 
     @staticmethod
-    def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
+    def count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
         system_blocks: Final = (
             sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0
         )
@@ -258,7 +260,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         ``max_blocks`` is reached. Injection points are honored in config order,
         so earlier points win when slots are scarce.
         """
-        used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages)
+        used_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
 
         limit_reached = False
         for point in points:
@@ -376,7 +378,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         # 2. list of objects - only apply to last item per Anthropic spec
         elif isinstance(message_content, list):
             if len(message_content) > 0 and isinstance(message_content[-1], dict):
-                message_content[-1]["cache_control"] = control
+                message_content[-1]["cache_control"] = control  # pyright: ignore[reportGeneralTypeIssues]  # loose runtime dict
         return message
 
     @staticmethod
@@ -454,8 +456,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
 
-        message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
-        system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system)
+        message_blocks: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
+        system_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints((), processed_system)
 
         if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks:
             system_already_has_cc: Final = isinstance(processed_system, list) and any(
@@ -589,7 +591,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         carry the mark either at the top level (Anthropic shape) or nested under
         ``function`` (OpenAI shape); the Anthropic chat transform accepts both.
         """
-        if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0:
+        if AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) > 0:
             return True
         if tools is not None:
             return any(
@@ -749,6 +751,64 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if points:
             non_default_params["cache_control_injection_points"] = points
 
+    @staticmethod
+    def record_gateway_injection(
+        request_kwargs: Mapping[str, object],
+        added: int,
+    ) -> None:
+        """Name the deployment whose payload the gateway, not the client, put breakpoints on.
+
+        Spend accounting only asks whether litellm acted, so what it needs is which
+        deployment, not a count. Recording that is what makes the mark attempt-scoped: the
+        metadata bucket is one dict shared by every retry, failover and fallback of a
+        request, and ``litellm_call_id`` is shared with it, so anything request-scoped
+        written by one attempt is read by all of them and each boundary would have to
+        remember to strip it. The deployment is the part that actually changes when the
+        request moves, so a leg that injected nothing is never credited for one that did.
+
+        It also makes a zero delta (hook re-entry) and a negative one (a prompt manager
+        replacing the messages) harmless, since neither rewrites an earlier mark.
+
+        A pass that runs before a deployment is chosen, which is what the proxy does for
+        prompt templates, injects into the payload every leg goes on to send, so it marks
+        the request for all of them rather than for one.
+
+        Only what this pass actually placed counts. A ``tool_config`` point is placed by
+        the Bedrock converse transform, and only when the request carries tools, so the
+        presence of one here says nothing about whether a breakpoint reaches the wire;
+        claiming it marked three request shapes out of four that inject nothing. Missing
+        that Bedrock credit is the fail-closed direction, and the alternative is a
+        provider transform that carries spend-attribution state.
+
+        Reads whichever bucket the request actually carries rather than asking the shared
+        name resolver, which answers on key presence: ``litellm_params`` declares
+        ``litellm_metadata`` as None on every request, so the resolver names a bucket that
+        is not there and the mark is dropped.
+
+        Never CREATES the bucket. The proxy seeds it on every request and is the marker's
+        only reader, so a request without one is a bare SDK call nothing would consume it
+        from. Creating it would also add a key to a dict call sites splat as ``**kwargs``,
+        and on the Responses API ``metadata`` is both this bucket's default name and an
+        explicit parameter, so the splat collides with the caller's own value.
+        """
+        if added <= 0:
+            return
+        bucket: Final = next(
+            (
+                candidate
+                for candidate in (request_kwargs.get("litellm_metadata"), request_kwargs.get("metadata"))
+                if isinstance(candidate, dict)
+            ),
+            None,
+        )
+        if bucket is not None:
+            model_info: Final = request_kwargs.get("model_info")
+            bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = (
+                model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT)
+                if isinstance(model_info, dict)
+                else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT
+            )
+
     @staticmethod
     def maybe_inject_cache_control(
         messages: list[dict],
@@ -798,17 +858,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
             model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
         )
-        breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system)
+        breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
         messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
             messages=messages,
             system=system,
             injection_points=injection_points,
             openai_dialect=openai_dialect,
         )
-        if (
-            openai_dialect
-            and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before
-        ):
+        breakpoints_added: Final = (
+            AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) - breakpoints_before
+        )
+        AnthropicCacheControlHook.record_gateway_injection(kwargs, breakpoints_added)
+        if openai_dialect and breakpoints_added > 0:
             kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
         if remaining:
             kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining)

litellm/integrations/langfuse/langfuse.py

@@ -5,6 +5,7 @@ import os
 import traceback
 from collections.abc import Callable, Iterable, Mapping
 from datetime import datetime
+from functools import lru_cache
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast
 
@@ -137,6 +138,16 @@ def resolve_langfuse_credentials(
     return public_key, secret_key, resolved_host
 
 
+@lru_cache(maxsize=8)
+def _warn_invalid_deployment_environment(raw_value: str, error: str) -> None:
+    verbose_logger.warning(
+        "Ignoring invalid LANGFUSE_TRACING_ENVIRONMENT=%r for the langfuse callback: %s. "
+        "Traces will be sent to Langfuse's default environment.",
+        raw_value,
+        error,
+    )
+
+
 class LangFuseLogger:
     # Class variables or attributes
     def __init__(
@@ -165,9 +176,11 @@ class LangFuseLogger:
             # add http:// if unset, assume communicating over private network - e.g. render
             self.langfuse_host = "http://" + self.langfuse_host
         _env_override: Final = str(langfuse_environment).strip() if langfuse_environment is not None else None
-        self.langfuse_environment = _env_override or os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
-        if self.langfuse_environment:
-            validate_langfuse_environment_value(self.langfuse_environment)
+        if _env_override:
+            validate_langfuse_environment_value(_env_override)
+            self.langfuse_environment: str | None = _env_override
+        else:
+            self.langfuse_environment = self.resolve_deployment_environment()
         self.langfuse_release = os.getenv("LANGFUSE_RELEASE")
         self.langfuse_debug = os.getenv("LANGFUSE_DEBUG")
         self.langfuse_flush_interval = LangFuseLogger._get_langfuse_flush_interval(flush_interval)
@@ -953,6 +966,20 @@ class LangFuseLogger:
             verbose_logger.warning("Failed to apply masking function: %s. Returning original data.", e)
             return data
 
+    @staticmethod
+    def resolve_deployment_environment() -> str | None:
+        """Resolve LANGFUSE_TRACING_ENVIRONMENT: stripped value, "default" plus a warning when invalid, None when unset."""
+        raw: Final = os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
+        if not raw:
+            return None
+        value: Final = raw.strip()
+        try:
+            validate_langfuse_environment_value(value)
+        except ValueError as e:
+            _warn_invalid_deployment_environment(raw, str(e))
+            return "default"
+        return value
+
     @staticmethod
     def _get_langfuse_flush_interval(flush_interval: int) -> int:
         """

litellm/integrations/langfuse/langfuse_handler.py

@@ -1,5 +1,3 @@
-import os
-
 """
 This file contains the LangFuseHandler class
 
@@ -8,6 +6,7 @@ Used to get the LangFuseLogger for a given request
 Handles Key/Team Based Langfuse Logging
 """
 
+import os
 from typing import TYPE_CHECKING, Any, Final
 
 from litellm.litellm_core_utils.litellm_logging import StandardCallbackDynamicParams
@@ -157,7 +156,11 @@ class LangFuseHandler:
         if raw is None:
             return None
         value = str(raw).strip()
-        if not value or value == os.getenv("LANGFUSE_TRACING_ENVIRONMENT"):
+        if (
+            not value
+            or value == os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
+            or value == LangFuseLogger.resolve_deployment_environment()
+        ):
             return None
         return value
 

litellm/integrations/langfuse/langfuse_prompt_management.py

@@ -2,6 +2,7 @@
 Call Hook for LiteLLM Proxy which allows Langfuse prompt management.
 """
 
+import inspect
 import os
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@@ -109,6 +110,9 @@ def langfuse_client_init(
             cert=os.getenv("SSL_CERTIFICATE", litellm.ssl_certificate),
         )
 
+    if "environment" in inspect.signature(Langfuse.__init__).parameters:
+        parameters["environment"] = LangFuseLogger.resolve_deployment_environment()
+
     client: Final = Langfuse(**parameters)
 
     return client

litellm/integrations/shadow_eval_logger.py

@@ -452,6 +452,14 @@ async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
     return False
 
 
+def _forwarded_team_id(metadata: Mapping[str, object]) -> str | None:
+    """The shadowed key's team, the identity the judge call already carries in its metadata
+    and the router already selects deployments with. Read here too so the arm choice, which
+    happens before the router sees the call, is made under the same team."""
+    team_id: Final = metadata.get("user_api_key_team_id")
+    return team_id if isinstance(team_id, str) and team_id else None
+
+
 def _routing_decision(metadata: Mapping[str, object]) -> Mapping[str, object]:
     """The routing decision a pre-routing strategy wrote to a call's metadata, empty when
     a plain model served it. Read off the sampled request for the control arm, and off the
@@ -915,6 +923,7 @@ class ShadowEvalLogger(CustomLogger):
                 self._router_provider(),
                 judge_model,
                 judge_messages,  # pyright: ignore[reportArgumentType]  # plain SDK message dicts
+                team_id=_forwarded_team_id(parent_metadata),
                 temperature=0,
                 max_tokens=JUDGE_MAX_OUTPUT_TOKENS,
                 response_format=PAIRWISE_JUDGE_RESPONSE_FORMAT,

litellm/litellm_core_utils/audio_utils/subtitle_utils.py

@@ -0,0 +1,193 @@
+"""Provider-agnostic SRT/WebVTT subtitle synthesis from timestamped transcription tokens."""
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from itertools import accumulate, chain
+from typing import Final
+
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
+
+CUE_MAX_TOKENS: Final = 15
+CUE_MAX_DURATION_MS: Final = 5000
+
+SRT_RESPONSE_FORMAT: Final = "srt"
+VTT_RESPONSE_FORMAT: Final = "vtt"
+SUBTITLE_RESPONSE_FORMATS: Final = frozenset((SRT_RESPONSE_FORMAT, VTT_RESPONSE_FORMAT))
+
+
+@dataclass(frozen=True, slots=True)
+class SubtitleToken:
+    text: str
+    start_ms: int | None = None
+    end_ms: int | None = None
+    speaker: str | int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class SubtitleCue:
+    start_ms: int
+    end_ms: int
+    text: str
+
+
+@dataclass(frozen=True, slots=True)
+class _CueAccumulator:
+    texts: tuple[str, ...] = ()
+    start_ms: int | None = None
+    end_ms: int | None = None
+    speaker: str | int | None = None
+
+
+def _completed_cue(accumulator: _CueAccumulator) -> tuple[SubtitleCue, ...]:
+    if not accumulator.texts or accumulator.start_ms is None:
+        return ()
+    text: Final = "".join(accumulator.texts).strip()
+    if not text:
+        return ()
+    end_ms: Final = accumulator.end_ms if accumulator.end_ms is not None else accumulator.start_ms
+    return (SubtitleCue(start_ms=accumulator.start_ms, end_ms=end_ms, text=text),)
+
+
+def _cue_break_reached(accumulator: _CueAccumulator, token: SubtitleToken) -> bool:
+    if len(accumulator.texts) >= CUE_MAX_TOKENS:
+        return True
+    return (
+        accumulator.start_ms is not None
+        and token.start_ms is not None
+        and token.start_ms - accumulator.start_ms >= CUE_MAX_DURATION_MS
+    )
+
+
+_AbsorbStep = tuple[tuple[SubtitleCue, ...], _CueAccumulator]
+
+
+def _absorb_token(accumulator: _CueAccumulator, token: SubtitleToken) -> _AbsorbStep:
+    if token.start_ms is None and accumulator.start_ms is None:
+        return (), accumulator
+    if token.speaker is not None and token.speaker != accumulator.speaker:
+        return _completed_cue(accumulator), _CueAccumulator(
+            texts=(token.text,),
+            start_ms=token.start_ms,
+            end_ms=token.end_ms,
+            speaker=token.speaker,
+        )
+    if _cue_break_reached(accumulator, token):
+        return _completed_cue(accumulator), _CueAccumulator(
+            texts=(token.text,),
+            start_ms=token.start_ms,
+            end_ms=token.end_ms,
+            speaker=accumulator.speaker,
+        )
+    return (), _CueAccumulator(
+        texts=(*accumulator.texts, token.text),
+        start_ms=accumulator.start_ms if accumulator.start_ms is not None else token.start_ms,
+        end_ms=token.end_ms if token.end_ms is not None else accumulator.end_ms,
+        speaker=accumulator.speaker,
+    )
+
+
+def _absorb_step(carry: _AbsorbStep, token: SubtitleToken) -> _AbsorbStep:
+    return _absorb_token(carry[1], token)
+
+
+def group_subtitle_tokens_into_cues(tokens: Sequence[SubtitleToken]) -> tuple[SubtitleCue, ...]:
+    steps: Final = tuple(accumulate(tokens, _absorb_step, initial=((), _CueAccumulator())))
+    completed: Final = chain.from_iterable(emitted for emitted, _ in steps)
+    return (*completed, *_completed_cue(steps[-1][1]))
+
+
+def _format_timestamp(total_ms: int, millis_separator: str) -> str:
+    clamped: Final = max(total_ms, 0)
+    hours, hour_remainder = divmod(clamped, 3_600_000)
+    minutes, minute_remainder = divmod(hour_remainder, 60_000)
+    seconds, millis = divmod(minute_remainder, 1_000)
+    return f"{hours:02d}:{minutes:02d}:{seconds:02d}{millis_separator}{millis:03d}"
+
+
+def _render_srt(cues: Sequence[SubtitleCue]) -> str:
+    lines: Final = tuple(
+        line
+        for index, cue in enumerate(cues, start=1)
+        for line in (
+            str(index),
+            f"{_format_timestamp(cue.start_ms, ',')} --> {_format_timestamp(cue.end_ms, ',')}",
+            cue.text,
+            "",
+        )
+    )
+    return "\n".join(lines)
+
+
+def _render_vtt(cues: Sequence[SubtitleCue]) -> str:
+    cue_lines: Final = tuple(
+        line
+        for cue in cues
+        for line in (
+            f"{_format_timestamp(cue.start_ms, '.')} --> {_format_timestamp(cue.end_ms, '.')}",
+            cue.text,
+            "",
+        )
+    )
+    return "\n".join(("WEBVTT", "", *cue_lines))
+
+
+def render_subtitle_tokens_as_srt(tokens: Sequence[SubtitleToken]) -> str:
+    """Render tokens as an SRT document; empty string when no token has timestamp data."""
+    cues: Final = group_subtitle_tokens_into_cues(tokens)
+    if not cues:
+        return ""
+    return _render_srt(cues)
+
+
+def render_subtitle_tokens_as_vtt(tokens: Sequence[SubtitleToken]) -> str:
+    """Render tokens as a WebVTT document; the WEBVTT header is emitted even without cues."""
+    return _render_vtt(group_subtitle_tokens_into_cues(tokens))
+
+
+class TranscriptionWordTiming(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    word: str = ""
+    start: float | None = None
+    end: float | None = None
+    speaker: str | None = None
+
+
+_WORD_TIMINGS_ADAPTER: Final = TypeAdapter(tuple[TranscriptionWordTiming, ...])
+
+
+def _seconds_to_ms(seconds: float | None) -> int | None:
+    if seconds is None:
+        return None
+    return round(seconds * 1000)
+
+
+def _word_to_subtitle_token(word: TranscriptionWordTiming) -> SubtitleToken:
+    return SubtitleToken(
+        text=f"{word.word} ",
+        start_ms=_seconds_to_ms(word.start),
+        end_ms=_seconds_to_ms(word.end),
+        speaker=word.speaker,
+    )
+
+
+def _parse_word_timings(words: object) -> tuple[TranscriptionWordTiming, ...]:
+    try:
+        return _WORD_TIMINGS_ADAPTER.validate_python(words)
+    except ValidationError:
+        return ()
+
+
+def synthesize_subtitle_document(words: object, response_format: str) -> str | None:
+    """
+    Build an SRT/VTT document from OpenAI verbose_json-style word dicts
+    (word/start/end in float seconds, optional speaker). Returns None when the
+    format is not a subtitle format or the words carry no usable timestamps.
+    """
+    if response_format not in SUBTITLE_RESPONSE_FORMATS:
+        return None
+    tokens: Final = tuple(_word_to_subtitle_token(word) for word in _parse_word_timings(words))
+    cues: Final = group_subtitle_tokens_into_cues(tokens)
+    if not cues:
+        return None
+    return _render_srt(cues) if response_format == SRT_RESPONSE_FORMAT else _render_vtt(cues)

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -2222,6 +2222,8 @@ def _map_exception_by_status(
     status_code: Final = original_exception.status_code if hasattr(original_exception, "status_code") else None
     if not isinstance(status_code, int) or status_code < 400:
         return
+    if getattr(original_exception, "status_code_is_synthesized", False):
+        return
     message: Final = f"{exception_provider} - {error_str}"
     response: Final = original_exception.response if hasattr(original_exception, "response") else None
     match status_code:

litellm/litellm_core_utils/litellm_logging.py

@@ -888,7 +888,10 @@ class Logging(LiteLLMLoggingBaseClass):
         prompt_management_logger: CustomLogger | None = None,
         prompt_label: str | None = None,
         prompt_version: int | None = None,
+        request_kwargs: dict[str, object] | None = None,  # mutable-ok: marker stamped into live request kwargs
     ) -> tuple[str, list[AllMessageValues], dict]:
+        from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+
         custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management(
             model=model,
             non_default_params=non_default_params,
@@ -898,6 +901,7 @@ class Logging(LiteLLMLoggingBaseClass):
         )
 
         if custom_logger:
+            breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
             (
                 model,
                 messages,
@@ -913,6 +917,11 @@ class Logging(LiteLLMLoggingBaseClass):
                 prompt_label=prompt_label,
                 prompt_version=prompt_version,
             )
+            if request_kwargs is not None:
+                AnthropicCacheControlHook.record_gateway_injection(
+                    request_kwargs,
+                    AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before,
+                )
         self.messages = messages
         return model, messages, non_default_params
 
@@ -928,7 +937,10 @@ class Logging(LiteLLMLoggingBaseClass):
         tools: list[dict] | None = None,
         prompt_label: str | None = None,
         prompt_version: int | None = None,
+        request_kwargs: dict[str, object] | None = None,  # mutable-ok: marker stamped into live request kwargs
     ) -> tuple[str, list[AllMessageValues], dict]:
+        from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+
         custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management(
             model=model,
             tools=tools,
@@ -939,6 +951,7 @@ class Logging(LiteLLMLoggingBaseClass):
         )
 
         if custom_logger:
+            breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
             (
                 model,
                 messages,
@@ -956,6 +969,11 @@ class Logging(LiteLLMLoggingBaseClass):
                 prompt_label=prompt_label,
                 prompt_version=prompt_version,
             )
+            if request_kwargs is not None:
+                AnthropicCacheControlHook.record_gateway_injection(
+                    request_kwargs,
+                    AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before,
+                )
         self.messages = messages
         return model, messages, non_default_params
 

litellm/litellm_core_utils/llm_judge.py

@@ -4,7 +4,9 @@ from __future__ import annotations
 
 import json
 import re
-from typing import TYPE_CHECKING, Final
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import TYPE_CHECKING, Final, Literal
 
 import litellm
 
@@ -56,17 +58,62 @@ def extract_text_from_content(content: object) -> str:
     return ""
 
 
-def router_resolves_model(router: Router | None, model: str) -> bool:
-    """Whether the model name resolves through the proxy's router (configured deployment
-    or model-group alias), the same check the judge dispatch itself makes, so start-time
-    validation cannot accept a name the call path then fails on."""
-    return router is not None and bool(model in router.model_group_alias or router.get_model_list(model_name=model))
+@lru_cache(maxsize=512)
+def _provider_qualified(model: str) -> str | None:
+    """`model` in the one spelling litellm itself resolves it to, or None if it maps to no
+    provider.
+
+    A deployment may be configured as `openai/gpt-4o` and a judge given as `gpt-4o`; both
+    reach the same model, so an identity that keeps them apart reports two models where
+    there is one. None is a different answer from "unchanged": a name that is already
+    provider-qualified normalises to itself, and reading that as a failure would call every
+    correctly-spelled public model unresolvable.
+    """
+    try:
+        stripped, provider, _, _ = litellm.get_llm_provider(model=model)
+    except Exception:  # noqa: BLE001  # an unmapped name has no provider, which is the answer
+        return None
+    return f"{provider}/{stripped}" if provider and stripped else None
+
+
+@dataclass(frozen=True, slots=True)
+class JudgeTarget:
+    """Where a call to one model name goes for one caller, and what answers it.
+
+    The single answer to that question: the resolvability gate, the judge-vs-candidate
+    gate and the dispatch all read it, so none of them can decide it differently. Splitting
+    it is what let start-time validation accept a team's own model while dispatch sent the
+    literal name to the SDK.
+    """
+
+    via: Literal["router", "sdk", "nothing"]
+    models: frozenset[str]
+
+
+def judge_target(router: Router | None, model: str, team_id: str | None = None) -> JudgeTarget:
+    """Resolve `model` the way a call from `team_id` would be.
+
+    Three outcomes and no others: the router serves it (a deployment, a team-public name,
+    an alias, a routing group or a wildcard, exactly the channels `get_model_list`
+    composes); the SDK serves it because litellm recognises the provider; or nothing does,
+    which is the only case a caller may refuse on.
+
+    `team_id` is part of the question, not a refinement of it. A team-public name resolves
+    only for its own team and a team's own deployment resolves for nobody else, so asking
+    without it answers for a caller who does not exist.
+    """
+    served: Final = router.resolved_litellm_models(model, team_id=team_id) if router is not None else ()
+    if served:
+        return JudgeTarget("router", frozenset(_provider_qualified(m) or m for m in served))
+    qualified: Final = _provider_qualified(model)
+    return JudgeTarget("sdk", frozenset({qualified})) if qualified is not None else JudgeTarget("nothing", frozenset())
 
 
 async def judge_acompletion(
     router: Router | None,
     judge_model: str,
     messages: list[AllMessageValues],  # mutable-ok: the SDK acompletion signature takes a list
+    team_id: str | None = None,
     **params: object,
 ) -> ModelResponse:
     """Dispatch a judge call through the proxy's router when the judge model is a
@@ -74,9 +121,13 @@ async def judge_acompletion(
     provider-qualified public names. The router path never retries or falls back:
     a failed judge call is the caller's counted failure, not a spend multiplier.
     Sampling preferences are advisory: models that removed sampling params (e.g.
-    claude-sonnet-5) drop them instead of rejecting the judge call."""
-    if router_resolves_model(router, judge_model):
-        return await router.acompletion(  # pyright: ignore[reportOptionalMemberAccess]  # router_resolves_model implies router is not None
+    claude-sonnet-5) drop them instead of rejecting the judge call.
+
+    The arm is chosen by `judge_target` under the caller's own team, the same call
+    start-time validation makes, so a judge a team can reach cannot be validated as a
+    deployment and then dispatched as a public name the SDK has never heard of."""
+    if judge_target(router, judge_model, team_id).via == "router":
+        return await router.acompletion(  # pyright: ignore[reportOptionalMemberAccess]  # a router target implies router is not None
             model=judge_model,
             messages=messages,
             num_retries=0,

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1747,6 +1747,46 @@ def hoist_images_from_tool_messages(
     ]
 
 
+def _is_tool_reference_part(part: object) -> bool:
+    return isinstance(part, dict) and part.get("type") == "tool_reference"
+
+
+def _tool_message_carries_tool_reference(message: AllMessageValues) -> bool:
+    if message.get("role") != "tool":
+        return False
+    content = message.get("content")
+    return isinstance(content, list) and any(_is_tool_reference_part(part) for part in content)
+
+
+def _drop_tool_reference_parts(message: AllMessageValues) -> AllMessageValues:
+    if not _tool_message_carries_tool_reference(message):
+        return message
+    content = cast(list, message.get("content"))  # cast-ok: shape checked by _tool_message_carries_tool_reference
+    remaining_parts = [  # mutable-ok: tool message content must stay a json list
+        part for part in content if not _is_tool_reference_part(part)
+    ]
+    new_content = remaining_parts if remaining_parts else ""
+    rewritten = {**message, "content": new_content}  # mutable-ok: chat messages are plain json dicts
+    return cast(AllMessageValues, rewritten)  # cast-ok: dict spread keeps keys like cache_control
+
+
+def drop_tool_reference_parts_from_tool_messages(
+    messages: list[AllMessageValues],  # mutable-ok: message pipelines type messages as mutable lists
+) -> list[AllMessageValues]:  # mutable-ok: message pipelines type messages as mutable lists
+    """
+    Remove tool_reference content parts from role:"tool" messages.
+
+    The OpenAI chat spec only accepts text in tool messages, so a tool_reference
+    part carried through the Anthropic adapter makes strict providers reject the
+    request. The reference names an already-declared tool rather than carrying
+    content, so it is dropped; a reference-only result keeps its tool message with
+    empty text so the preceding tool_call stays answered.
+    """
+    if not any(_tool_message_carries_tool_reference(message) for message in messages):
+        return messages
+    return [_drop_tool_reference_parts(message) for message in messages]  # mutable-ok: pipelines mutate message lists
+
+
 def _attempt_json_repair(s: str) -> Any | None:
     """
     Attempt to repair truncated JSON produced by LLM tool calls.

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1412,7 +1412,7 @@ def convert_to_gemini_tool_call_result(
                             )
                         except Exception as e:
                             verbose_logger.warning("Failed to process image in tool response: %s", e)
-                elif content_type in ("file", "input_file"):
+                elif content_type in ("file", "input_file"):  # pyright: ignore[reportUnnecessaryContains]  # loose runtime dict
                     # Extract file for inline_data (for tool results with PDF, audio, video, etc.)
                     file_data = content.get("file_data", "")
                     if not file_data:
@@ -1564,14 +1564,23 @@ def convert_to_anthropic_tool_result(
     }
     """
     anthropic_content: (
-        str | list[AnthropicMessagesToolResultContent | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam]
+        str
+        | list[
+            AnthropicMessagesToolResultContent
+            | AnthropicMessagesImageParam
+            | AnthropicMessagesDocumentParam
+            | ToolReference
+        ]
     ) = ""
     if isinstance(message["content"], str):
         anthropic_content = message["content"]
     elif isinstance(message["content"], list):
         content_list: Final = message["content"]
         anthropic_content_list: list[
-            AnthropicMessagesToolResultContent | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam
+            AnthropicMessagesToolResultContent
+            | AnthropicMessagesImageParam
+            | AnthropicMessagesDocumentParam
+            | ToolReference
         ] = []
         for content in content_list:
             if content["type"] == "text":
@@ -1614,6 +1623,8 @@ def convert_to_anthropic_tool_result(
                         original_content_element=content,
                     )
                     anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
+            elif content["type"] == "tool_reference":
+                anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
             elif content["type"] == "file":
                 file_content = cast(ChatCompletionFileObject, content)
                 _file_block = anthropic_process_openai_file_message(file_content)

litellm/litellm_core_utils/streaming_handler.py

@@ -8,11 +8,12 @@ import time
 import traceback
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
+from types import MappingProxyType
 from typing import Any, Final, NoReturn, Protocol, TypeVar, cast
 
 import anyio
 import httpx
-from pydantic import BaseModel
+from pydantic import BaseModel, ValidationError
 from typing_extensions import NotRequired, TypedDict
 
 import litellm
@@ -182,6 +183,23 @@ class _VertexChunkLike(Protocol):
     candidates: Sequence[_VertexCandidateLike]
 
 
+class _ParsedChunkHiddenParams(BaseModel):
+    provider_specific_fields: Mapping[str, object] | None = None
+
+
+def _provider_hidden_params(chunk: object) -> Mapping[str, object] | None:
+    hidden: Final[object] = getattr(chunk, "_hidden_params", None)
+    if not isinstance(hidden, dict):
+        return None
+    try:
+        parsed: Final = _ParsedChunkHiddenParams.model_validate(hidden)
+    except ValidationError:
+        return None
+    if not parsed.provider_specific_fields:
+        return None
+    return MappingProxyType({"provider_specific_fields": dict(parsed.provider_specific_fields)})
+
+
 class CustomStreamWrapper:
     def __init__(
         self,
@@ -801,7 +819,7 @@ class CustomStreamWrapper:
         except Exception as e:
             raise e
 
-    def model_response_creator(self, chunk: dict | None = None, hidden_params: dict | None = None):
+    def model_response_creator(self, chunk: dict | None = None, hidden_params: Mapping[str, object] | None = None):
         _model: Final = self._cached_model_name
         _logging_obj_llm_provider: Final = self._cached_logging_llm_provider
 
@@ -1504,7 +1522,7 @@ class CustomStreamWrapper:
     def chunk_creator(self, chunk: Any):
         if hasattr(chunk, "id"):
             self.response_id = chunk.id
-        model_response = self.model_response_creator()
+        model_response = self.model_response_creator(hidden_params=_provider_hidden_params(chunk))
         response_obj: dict[str, Any] = {}
         try:
             # return this for all models

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -24,10 +24,12 @@ from litellm._logging import verbose_proxy_logger
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
     LiteLLMAnthropicMessagesAdapter,
+    is_provider_native_tool_dict,
 )
 from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
 from litellm.llms.base_llm.guardrail_translation.utils import (
     anthropic_tool_name,
+    anthropic_tool_names,
     effective_scan_only_tool_results_for_guardrail,
     effective_skip_system_message_for_guardrail,
     effective_skip_tool_message_for_guardrail,
@@ -360,7 +362,13 @@ class AnthropicMessagesHandler(BaseTranslation):
         structured_messages: Final = [full_structured_messages[index] for index in scoped_message_indices]
 
         tools_to_check: Final[list[ChatCompletionToolParam]] = (
-            [] if scan_only_tool_results else chat_completion_compatible_request.get("tools", [])
+            []
+            if scan_only_tool_results
+            else [
+                tool
+                for tool in chat_completion_compatible_request.get("tools", [])
+                if not is_provider_native_tool_dict(tool)
+            ]
         )
 
         # Step 1: Extract all text content and images
@@ -419,7 +427,10 @@ class AnthropicMessagesHandler(BaseTranslation):
                         tool_name=anthropic_tool_name,
                     )
                     if scan_only_tool_results
-                    else anthropic_tools
+                    else [
+                        *(tool for tool in data.get("tools") or [] if is_provider_native_tool_dict(tool)),
+                        *anthropic_tools,
+                    ]
                 )
 
             guardrailed_structured_messages: Final = guardrailed_inputs.get("structured_messages")
@@ -677,12 +688,9 @@ class AnthropicMessagesHandler(BaseTranslation):
         )
 
     def extract_request_tool_names(self, data: dict) -> list[str]:
-        """Extract tool names from Anthropic messages request (tools[].name)."""
-        names: Final[list[str]] = []
-        for tool in data.get("tools") or []:
-            if isinstance(tool, dict) and tool.get("name"):
-                names.append(str(tool["name"]))
-        return names
+        """Extract every tool name in an Anthropic messages request: tools[].name, plus
+        tools[].function.name for OpenAI-format tools the bridge forwards verbatim."""
+        return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)]
 
     @classmethod
     def _extract_input_text_and_images(

litellm/llms/anthropic/common_utils.py

@@ -974,19 +974,25 @@ def strip_advisor_blocks_from_messages(messages: list[Any], replace_with_text: b
     return messages
 
 
-def is_anthropic_invalid_thinking_signature_error(error_text: str) -> bool:
+def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool:
     """
-    Detect Anthropic 400 errors caused by missing or invalid thinking signatures.
+    Detect Anthropic 400 errors caused by invalid thinking blocks in replayed
+    history: a missing or invalid signature, or a block with empty thinking text.
 
     Known error formats:
     {"message":"messages.2.content.0.thinking.signature.str: Input should be a valid string"}
     messages.N.content.M.thinking.signature.str: Input should be a valid string
     messages.N.content.M: Invalid `signature` in `thinking` block
+    messages.N.content.M.thinking: each thinking block must contain thinking
     """
     if not error_text:
         return False
     lower: Final = error_text.lower()
-    return "thinking" in lower and "signature" in lower and ("invalid" in lower or "valid string" in lower)
+    if "thinking" not in lower:
+        return False
+    if "signature" in lower and ("invalid" in lower or "valid string" in lower):
+        return True
+    return "must contain thinking" in lower
 
 
 def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[Any]:
@@ -1028,22 +1034,29 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(
     data.pop("thinking", None)
 
 
-def strip_empty_text_blocks_from_anthropic_messages(
+def strip_empty_content_blocks_from_anthropic_messages(
     messages: list[Any],
 ) -> list[Any]:
     """
     Return a new message list with empty or whitespace-only ``{"type": "text"}``
-    content blocks removed.
+    and ``{"type": "thinking"}`` content blocks removed.
 
     Anthropic's API rejects requests containing such blocks with
-    ``"messages: text content blocks must be non-empty"``, but assistant
-    messages from Anthropic routinely arrive with ``{"type": "text", "text": ""}``
-    alongside ``tool_use`` blocks (see anthropics/anthropic-sdk-python#461).
+    ``"messages: text content blocks must be non-empty"`` and
+    ``"messages.N.content.M.thinking: each thinking block must contain
+    thinking"`` respectively.  Assistant messages routinely arrive with
+    ``{"type": "text", "text": ""}`` alongside ``tool_use`` blocks (see
+    anthropics/anthropic-sdk-python#461), and a turn served by a
+    non-Anthropic reasoning model through the /v1/messages bridge can carry
+    ``{"type": "thinking", "thinking": ""}`` when the model produced no
+    reasoning text (e.g. it went straight to parallel tool calls).
     Multi-turn tool-use clients (e.g. Claude Code) loop these prior responses
     back as conversation history, which then causes the next request to 400
     on the unified ``/v1/messages`` path.  ``/v1/chat/completions`` already
     handles this in ``anthropic_messages_pt``; this helper provides the
     equivalent guarantee for the native Anthropic Messages path.
+    ``redacted_thinking`` blocks are never touched: they carry opaque
+    ``data`` instead of thinking text.
 
     Messages whose content is a list and becomes empty after stripping are
     omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`.
@@ -1056,7 +1069,7 @@ def strip_empty_text_blocks_from_anthropic_messages(
             out.append(m)
             continue
         content = m["content"]
-        filtered = [b for b in content if not _is_empty_text_block(b)]
+        filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_thinking_block(b)]
         if len(filtered) == len(content):
             out.append(m)
         elif filtered:
@@ -1071,6 +1084,21 @@ def _is_empty_text_block(block: Any) -> bool:
     return not isinstance(text, str) or not text.strip()
 
 
+def is_empty_thinking_block(block: object) -> bool:
+    """
+    True for a ``{"type": "thinking"}`` content block whose thinking text is
+    missing, not a string, or empty/whitespace-only after ``.strip()``.
+    Anthropic rejects such blocks with ``"each thinking block must contain
+    thinking"`` (whitespace-only included, verified live), regardless of any
+    signature they carry.  ``redacted_thinking`` blocks are a different type
+    and always return False.
+    """
+    if not isinstance(block, dict) or block.get("type") != "thinking":
+        return False
+    thinking: Final = block.get("thinking")
+    return not isinstance(thinking, str) or not thinking.strip()
+
+
 def normalize_anthropic_tool_use_id(raw_id: str) -> str:
     """
     Normalize a tool_use / tool_result id for Anthropic's ``^[a-zA-Z0-9_-]+$``

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -1029,6 +1029,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
 
     @staticmethod
     def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
+        from litellm.llms.anthropic.common_utils import is_empty_thinking_block
+
         choice: Final = chunk.choices[0]
         if choice.finish_reason is not None:
             return False
@@ -1039,7 +1041,11 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             return False
         if getattr(delta, "reasoning_content", None):
             return False
-        if getattr(delta, "thinking_blocks", None):
+        # thinking_blocks whose entries are all empty (even if signed) must not
+        # open a block: the emitted {"type": "thinking", "thinking": ""} gets
+        # replayed as history and Anthropic rejects it (LIT-6357).
+        thinking_blocks: Final = getattr(delta, "thinking_blocks", None)
+        if thinking_blocks and any(isinstance(b, dict) and not is_empty_thinking_block(b) for b in thinking_blocks):
             return False
         return True
 

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -1,8 +1,8 @@
 import copy
 import hashlib
 import json
-from collections.abc import AsyncIterator, Iterator, Mapping
-from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
+from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
 
 import litellm
 from litellm.llms.anthropic.experimental_pass_through.utils import (
@@ -18,6 +18,22 @@ TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LE
 PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
 
 
+_ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset(
+    {"name", "type", "input_schema", "description", "cache_control", "strict"}
+)
+
+
+def _is_openai_function_tool(tool: Mapping[str, object]) -> bool:
+    return tool.get("type") == "function" and "function" in tool
+
+
+def is_provider_native_tool_dict(tool: Mapping[str, object]) -> bool:
+    if len(tool) != 1:
+        return False
+    key, value = next(iter(tool.items()))
+    return key not in _ANTHROPIC_TOOL_SCHEMA_KEYS and isinstance(value, dict)
+
+
 def truncate_tool_name(name: str) -> str:
     """
     Truncate tool names that exceed OpenAI's 64-character limit.
@@ -73,7 +89,10 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
 from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking_budget,
 )
-from litellm.llms.anthropic.common_utils import normalize_anthropic_tool_use_id
+from litellm.llms.anthropic.common_utils import (
+    is_empty_thinking_block,
+    normalize_anthropic_tool_use_id,
+)
 from litellm.llms.anthropic.experimental_pass_through.context_management import (
     PolyfillResult,
 )
@@ -126,7 +145,9 @@ from litellm.types.llms.openai import (
     ChatCompletionToolMessage,
     ChatCompletionToolParam,
     ChatCompletionToolParamFunctionChunk,
+    ChatCompletionToolReferenceObject,
     ChatCompletionUserMessage,
+    ToolMessageContentPart,
 )
 from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
 
@@ -135,6 +156,8 @@ from .streaming_iterator import AnthropicStreamWrapper
 if TYPE_CHECKING:
     from litellm.types.llms.anthropic import ContentBlockContentBlockDict
 
+ToolResultContent: TypeAlias = str | list[ToolMessageContentPart]
+
 
 class AnthropicAdapter:
     def __init__(self) -> None:
@@ -412,90 +435,13 @@ class LiteLLMAnthropicMessagesAdapter:
                                 self._add_cache_control_if_applicable(content, doc_obj, model)
                                 new_user_content_list.append(doc_obj)
                         elif content.get("type") == "tool_result":
-                            if "content" not in content:
-                                tool_result = ChatCompletionToolMessage(
-                                    role="tool",
-                                    tool_call_id=content.get("tool_use_id", ""),
-                                    content="",
-                                )
-                                self._add_cache_control_if_applicable(content, tool_result, model)
-                                tool_message_list.append(tool_result)
-                            elif isinstance(content.get("content"), str):
-                                tool_result = ChatCompletionToolMessage(
-                                    role="tool",
-                                    tool_call_id=content.get("tool_use_id", ""),
-                                    content=str(content.get("content", "")),
-                                )
-                                self._add_cache_control_if_applicable(content, tool_result, model)
-                                tool_message_list.append(tool_result)
-                            elif isinstance(content.get("content"), list):
-                                # Combine all content items into a single tool message
-                                # to avoid creating multiple tool_result blocks with the same ID
-                                # (each tool_use must have exactly one tool_result)
-                                content_items = list(content.get("content", []))
-
-                                # Single-item text keeps the backward-compatible string format; a single
-                                # image or document becomes a structured image_url part
-                                if len(content_items) == 1:
-                                    c = content_items[0]
-                                    if isinstance(c, str):
-                                        tool_result = ChatCompletionToolMessage(
-                                            role="tool",
-                                            tool_call_id=content.get("tool_use_id", ""),
-                                            content=c,
-                                        )
-                                        self._add_cache_control_if_applicable(content, tool_result, model)
-                                        tool_message_list.append(tool_result)
-                                    elif isinstance(c, dict):
-                                        if c.get("type") == "text":
-                                            tool_result = ChatCompletionToolMessage(
-                                                role="tool",
-                                                tool_call_id=content.get("tool_use_id", ""),
-                                                content=c.get("text", ""),
-                                            )
-                                            self._add_cache_control_if_applicable(content, tool_result, model)
-                                            tool_message_list.append(tool_result)
-                                        elif c.get("type") in ("image", "document"):
-                                            image_part = self._tool_result_image_part(c.get("source"))
-                                            tool_result = ChatCompletionToolMessage(
-                                                role="tool",
-                                                tool_call_id=content.get("tool_use_id", ""),
-                                                content=[image_part]  # mutable-ok: content must be a json list
-                                                if image_part
-                                                else "",
-                                            )
-                                            self._add_cache_control_if_applicable(content, tool_result, model)
-                                            tool_message_list.append(tool_result)
-                                else:
-                                    # For multiple content items, combine into a single tool message
-                                    # with list content to preserve all items while having one tool_use_id
-                                    combined_content_parts: list[
-                                        ChatCompletionTextObject | ChatCompletionImageObject
-                                    ] = []
-                                    for c in content_items:
-                                        if isinstance(c, str):
-                                            combined_content_parts.append(ChatCompletionTextObject(type="text", text=c))
-                                        elif isinstance(c, dict):
-                                            if c.get("type") == "text":
-                                                combined_content_parts.append(
-                                                    ChatCompletionTextObject(
-                                                        type="text",
-                                                        text=c.get("text", ""),
-                                                    )
-                                                )
-                                            elif c.get("type") in ("image", "document"):
-                                                image_part = self._tool_result_image_part(c.get("source"))
-                                                if image_part:
-                                                    combined_content_parts.append(image_part)
-                                    # Create a single tool message with combined content
-                                    if combined_content_parts:
-                                        tool_result = ChatCompletionToolMessage(
-                                            role="tool",
-                                            tool_call_id=content.get("tool_use_id", ""),
-                                            content=combined_content_parts,
-                                        )
-                                        self._add_cache_control_if_applicable(content, tool_result, model)
-                                        tool_message_list.append(tool_result)
+                            tool_result = ChatCompletionToolMessage(
+                                role="tool",
+                                tool_call_id=content.get("tool_use_id", ""),
+                                content=self._tool_result_content(content.get("content")),
+                            )
+                            self._add_cache_control_if_applicable(content, tool_result, model)
+                            tool_message_list.append(tool_result)
 
             if len(tool_message_list) > 0:
                 new_messages.extend(tool_message_list)
@@ -771,6 +717,10 @@ class LiteLLMAnthropicMessagesAdapter:
                 new_tools.append(tool)
                 continue
 
+            if _is_openai_function_tool(tool) or is_provider_native_tool_dict(tool):
+                new_tools.append(cast(ChatCompletionToolParam, tool))  # cast-ok: passed through verbatim to provider
+                continue
+
             raw_name = tool.get("name")
             if raw_name is None or (isinstance(raw_name, str) and not str(raw_name).strip()):
                 original_name = f"litellm_unnamed_tool_{idx}"
@@ -943,6 +893,31 @@ class LiteLLMAnthropicMessagesAdapter:
         )
         return "prompt_cache_key" in (supported_params or ())
 
+    @staticmethod
+    def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool:
+        """Whether the target declares ``reasoning_effort`` among its supported params.
+
+        A Claude-family target is recognized by name, which says nothing about the carrier the
+        provider serving it accepts: Snowflake serves Claude over the Anthropic dialect and
+        declares ``thinking`` alone, so storing the tier there raises before the request reaches
+        the wire.
+
+        Without a resolved provider the tier stays behind, which is what this bridge sent before
+        it carried one at all. Reading the declaration from the model's own prefix instead would
+        resolve the provider through a lookup that runs an OAuth device flow for two of them, and
+        this runs inside a logging callback as well as on the request path.
+
+        Unlike ``_supports_prompt_cache_key`` this does not exclude a provider that proxies an
+        unknown backend, because that provider declares this param and forwards it to a proxy
+        that resolves the real target itself, where a derived cache key has no such guarantee.
+        """
+        if not model or not custom_llm_provider:
+            return False
+        supported_params: Final = litellm.get_supported_openai_params(
+            model=model, custom_llm_provider=custom_llm_provider
+        )
+        return "reasoning_effort" in (supported_params or ())
+
     def _translate_metadata_to_openai(
         self,
         anthropic_message_request: AnthropicMessagesRequest,
@@ -1031,8 +1006,32 @@ class LiteLLMAnthropicMessagesAdapter:
         self,
         anthropic_message_request: AnthropicMessagesRequest,
         new_kwargs: ChatCompletionRequest,
+        *,
+        custom_llm_provider: str | None = None,
     ) -> None:
-        """Translate Anthropic thinking to either thinking or reasoning_effort."""
+        """Translate Anthropic thinking to either thinking or reasoning_effort.
+
+        A Claude-family target keeps ``thinking`` verbatim, since every bridged provider serving one
+        speaks that param. Carrying its adaptive effort tier alongside takes two different params,
+        because the two are not interchangeable at the provider mapping below.
+
+        Bedrock takes ``output_config`` directly, which attaches the tier and leaves ``thinking``
+        alone. Another bridged Claude target takes ``reasoning_effort`` if it declares that param,
+        and used to be sent no tier at all, so an adaptive request arrived byte-identical whichever
+        effort the caller asked for. That tier stays a plain string there, since the summary it
+        would otherwise be wrapped with already travels inside the forwarded ``thinking`` block,
+        and the wrapped dict is rejected outright by some of these providers.
+
+        A target declaring neither carrier keeps its bare ``thinking`` block. Being Claude-family
+        is a fact about the model, not about the params the provider in front of it accepts, so
+        the tier is offered only where the target says it is taken.
+
+        ``reasoning_effort`` is not a substitute for ``output_config`` on the Bedrock side: an
+        application inference profile ARN resolves to neither, so the tier is dropped, and providers
+        that rebuild ``output_config`` from it overwrite a caller-set ``thinking.display`` doing so.
+        An adaptive request with no tier stays untouched either way, so the provider's own default
+        still applies.
+        """
         if "thinking" not in anthropic_message_request:
             return
 
@@ -1041,35 +1040,40 @@ class LiteLLMAnthropicMessagesAdapter:
             return
 
         model: Final = new_kwargs.get("model", "")
-        if self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model):
+        is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model(
+            model
+        )
+        is_claude_target: Final = self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)
+        output_config: Final = anthropic_message_request.get("output_config")
+
+        if is_claude_target:
             new_kwargs["thinking"] = thinking
-            # Adaptive thinking without its effort tier makes Bedrock Converse
-            # return zero reasoning blocks, so forward output_config (minus
-            # `format`, already translated to response_format) for Bedrock
-            # targets only: other bridged providers reject the raw param, and
-            # get_llm_provider strips the `bedrock/` prefix before this runs.
-            if model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model(model):
-                claude_output_config: Final = anthropic_message_request.get("output_config")
-                if isinstance(claude_output_config, dict):
-                    effort_config: Final = {k: v for k, v in claude_output_config.items() if k != "format"}
+            if is_bedrock_target:
+                if isinstance(output_config, dict):
+                    effort_config: Final = {k: v for k, v in output_config.items() if k != "format"}
                     if effort_config:
                         new_kwargs["output_config"] = effort_config  # rebind-ok: out-param store like thinking above
+                return
+            if not self._target_declares_reasoning_effort(model, custom_llm_provider):
+                return
+
+        thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None
+        declared_effort: Final = (
+            output_config.get("effort") if thinking_type == "adaptive" and isinstance(output_config, dict) else None
+        )
+        if is_claude_target and not declared_effort:
             return
 
-        reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(cast(AnthropicThinkingParam, thinking))
+        reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort(
+            cast(AnthropicThinkingParam, thinking)
+        )
         if not reasoning_effort:
             return
 
-        thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None
-
-        # For adaptive thinking, override with output_config.effort if available
-        if thinking_type == "adaptive":
-            output_config: Final = anthropic_message_request.get("output_config")
-            if isinstance(output_config, dict) and output_config.get("effort"):
-                reasoning_effort = output_config["effort"]
-
-        new_kwargs["reasoning_effort"] = self._apply_reasoning_summary_wrapping(
-            reasoning_effort, cast(dict[str, object], thinking)
+        new_kwargs["reasoning_effort"] = (
+            reasoning_effort
+            if is_claude_target
+            else self._apply_reasoning_summary_wrapping(reasoning_effort, cast(dict[str, object], thinking))
         )
 
     def _translate_output_format_to_openai(
@@ -1165,6 +1169,7 @@ class LiteLLMAnthropicMessagesAdapter:
         self._translate_thinking_to_openai(
             anthropic_message_request=anthropic_message_request,
             new_kwargs=new_kwargs,
+            custom_llm_provider=custom_llm_provider,
         )
         ## CONVERT STOP_SEQUENCES
         self._translate_stop_sequences_to_openai(
@@ -1210,6 +1215,39 @@ class LiteLLMAnthropicMessagesAdapter:
 
         return None
 
+    def _tool_result_content(self, raw_content: object) -> ToolResultContent:
+        if isinstance(raw_content, str):
+            return raw_content
+        if not isinstance(raw_content, list):
+            return ""
+        items: Final = cast(Sequence[object], raw_content)  # cast-ok: untrusted client payload
+        parts: Final = tuple(part for part in (self._tool_result_part(item) for item in items) if part is not None)
+        match parts:
+            case ():
+                return ""
+            case ({"type": "text", "text": str(text)},):
+                return text
+            case _:
+                return list(parts)  # mutable-ok: content must be a json list
+
+    def _tool_result_part(self, item: object) -> ToolMessageContentPart | None:
+        if isinstance(item, str):
+            return ChatCompletionTextObject(type="text", text=item)
+        if not isinstance(item, dict):
+            return None
+        block: Final = cast(Mapping[str, object], item)  # cast-ok: untrusted client payload
+        match block.get("type"):
+            case "text":
+                return ChatCompletionTextObject(type="text", text=str(block.get("text") or ""))
+            case "image" | "document":
+                return self._tool_result_image_part(block.get("source"))
+            case "tool_reference":
+                return ChatCompletionToolReferenceObject(
+                    type="tool_reference", tool_name=str(block.get("tool_name") or "")
+                )
+            case _:
+                return None
+
     def _tool_result_image_part(self, image_source: object) -> ChatCompletionImageObject | None:
         if not isinstance(image_source, dict):
             return None
@@ -1229,6 +1267,8 @@ class LiteLLMAnthropicMessagesAdapter:
             if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
                 for thinking_block in choice.message.thinking_blocks:
                     if thinking_block.get("type") == "thinking":
+                        if is_empty_thinking_block(thinking_block):
+                            continue
                         thinking_value = thinking_block.get("thinking", "")
                         signature_value = thinking_block.get("signature", "")
                         new_content.append(

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -17,7 +17,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.llms.anthropic.common_utils import (
     flatten_unencrypted_web_search_results_in_anthropic_messages,
     sanitize_tool_use_ids_in_anthropic_messages,
-    strip_empty_text_blocks_from_anthropic_messages,
+    strip_empty_content_blocks_from_anthropic_messages,
 )
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
@@ -242,17 +242,20 @@ async def anthropic_messages(
     """
     Async: Make llm api request in Anthropic /messages API spec.
 
-    Runs the empty-text-block sanitizer before any backend dispatch.
+    Runs the empty-content-block sanitizer before any backend dispatch.
     """
     # Anthropic's API rejects requests containing empty / whitespace-only
-    # text content blocks with "messages: text content blocks must be
-    # non-empty".  Multi-turn tool-use clients (e.g. Claude Code) routinely
-    # loop assistant responses that contain {"type": "text", "text": ""}
-    # alongside tool_use blocks back as conversation history, which then
-    # causes the next /v1/messages call to 400.  /v1/chat/completions
-    # already handles this in anthropic_messages_pt; sanitize the native
-    # Anthropic Messages path here for the same guarantee.  See #22930.
-    messages = strip_empty_text_blocks_from_anthropic_messages(messages)
+    # text content blocks ("messages: text content blocks must be
+    # non-empty") and empty thinking blocks ("each thinking block must
+    # contain thinking").  Multi-turn tool-use clients (e.g. Claude Code)
+    # routinely loop assistant responses that contain such blocks — an empty
+    # text block alongside tool_use, or an empty thinking block from a turn
+    # a non-Anthropic reasoning model served through the bridge — back as
+    # conversation history, which then causes the next /v1/messages call to
+    # 400.  /v1/chat/completions already handles this in
+    # anthropic_messages_pt; sanitize the native Anthropic Messages path
+    # here for the same guarantee.  See #22930.
+    messages = strip_empty_content_blocks_from_anthropic_messages(messages)
     # Replay of cross-provider tool history (e.g. kimi -> Anthropic) may carry
     # ids like ``functions.Bash:0`` that violate Anthropic's id pattern.
     messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
@@ -374,7 +377,7 @@ async def anthropic_messages(
         api_base=api_base,
         client=client,
         custom_llm_provider=custom_llm_provider,
-        # messages were already empty-text-block sanitized at the top of this
+        # messages were already empty-content-block sanitized at the top of this
         # function and are NOT reassigned before this dispatch, so the handler
         # can skip its (otherwise redundant) second full-messages scan. Passed
         # explicitly (not via **kwargs) so it only affects this direct
@@ -451,7 +454,7 @@ def anthropic_messages_handler(
     # ``_litellm_messages_presanitized`` to skip this redundant second
     # full-messages scan. Pop it so it never leaks into provider params.
     if not kwargs.pop("_litellm_messages_presanitized", False):
-        messages = strip_empty_text_blocks_from_anthropic_messages(messages)
+        messages = strip_empty_content_blocks_from_anthropic_messages(messages)
         messages = sanitize_tool_use_ids_in_anthropic_messages(messages)
         messages = flatten_unencrypted_web_search_results_in_anthropic_messages(messages)
 

litellm/llms/anthropic/experimental_pass_through/utils.py

@@ -1,4 +1,6 @@
 import os
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import Final
 
 import litellm
@@ -6,6 +8,15 @@ from litellm.types.utils import ModelInfo
 
 OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH: Final = 64
 
+_EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
+    {
+        "max": ("max", "xhigh", "high"),
+        "xhigh": ("xhigh", "high"),
+        "minimal": ("minimal", "low"),
+    }
+)
+_THINKING_OFF: Final = "none"
+
 
 def prompt_cache_key_from_user_id(user_id: object) -> str | None:
     if user_id is None:
@@ -28,38 +39,33 @@ def normalize_reasoning_effort_value(
     model: str,
     custom_llm_provider: str | None = None,
 ) -> str:
-    """
-    Normalize a reasoning effort value based on model capabilities.
+    """Lower a tier the deployment does not accept to the nearest one it does, leaving others alone.
 
-    Degradation chains:
-    - "max"     → max / xhigh / high
-    - "xhigh"   → xhigh / high
-    - "minimal" → minimal / low
-    - other values pass through unchanged
+    The accepted set is resolved by the same owner that answers ``/model_group/info``, so a level
+    the proxy advertises is a level this path forwards.
+
+    A deployment that refuses every step of a chain falls back to an accepted level read off that
+    same set rather than to an assumed one, since an entry naming its levels outright can exclude
+    the tiers the per-level flags treat as unconditional. ``none`` is never that fallback and is
+    never degraded to, being an off switch rather than a tier; an always-on-thinking model is
+    handled where the thinking block is built. A deployment accepting no tier at all keeps the
+    chain's floor, which is what every deployment degraded to before there was anything to ask.
     """
-    if effort not in ("max", "xhigh", "minimal"):
+    chain: Final = _EFFORT_DEGRADATION_CHAIN.get(effort)
+    if chain is None:
         return effort
 
+    from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
     from litellm.utils import get_model_info
 
-    model_info: ModelInfo | None = None
     try:
-        model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
+        model_info: Final[ModelInfo] = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
     except Exception:
-        model_info = None
+        return chain[-1]
 
-    if effort == "max":
-        if model_info and model_info.get("supports_max_reasoning_effort"):
-            return "max"
-        if model_info and model_info.get("supports_xhigh_reasoning_effort"):
-            return "xhigh"
-        return "high"
-    elif effort == "xhigh":
-        if model_info and model_info.get("supports_xhigh_reasoning_effort"):
-            return "xhigh"
-        return "high"
-    elif effort == "minimal":
-        if model_info and model_info.get("supports_minimal_reasoning_effort"):
-            return "minimal"
-        return "low"
-    return "medium"
+    supported: Final = resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True)
+    if not supported:
+        return chain[-1]
+
+    accepted_tiers: Final = tuple(level for level in supported if level != _THINKING_OFF)
+    return next((level for level in (*chain, *accepted_tiers) if level in supported), chain[-1])

litellm/llms/azure/chat/gpt_transformation.py

@@ -4,6 +4,7 @@ from httpx._models import Headers, Response
 
 import litellm
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    drop_tool_reference_parts_from_tool_messages,
     hoist_images_from_tool_messages,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -252,7 +253,8 @@ class AzureOpenAIConfig(BaseConfig):
         litellm_params: dict,
         headers: dict,
     ) -> dict:
-        azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(messages))
+        stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(messages)
+        azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages))
         return {
             "model": model,
             "messages": azure_messages,

litellm/llms/base_llm/anthropic_messages/transformation.py

@@ -159,20 +159,20 @@ class BaseAnthropicMessagesConfig(ABC):
         and issue one more attempt (bounded by max_retry_on_anthropic_messages_http_error).
         """
         from litellm.llms.anthropic.common_utils import (
-            is_anthropic_invalid_thinking_signature_error,
+            is_anthropic_invalid_thinking_block_error,
         )
 
-        return e.response.status_code == 400 and is_anthropic_invalid_thinking_signature_error(e.response.text)
+        return e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text)
 
     def transform_anthropic_messages_request_on_http_error(self, e: httpx.HTTPStatusError, request_data: dict) -> dict:
         """
         Mutates request_data in place when retrying after a recoverable HTTP error.
         """
         from litellm.llms.anthropic.common_utils import (
-            is_anthropic_invalid_thinking_signature_error,
+            is_anthropic_invalid_thinking_block_error,
             strip_thinking_blocks_from_anthropic_messages_request_dict,
         )
 
-        if e.response.status_code == 400 and is_anthropic_invalid_thinking_signature_error(e.response.text):
+        if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text):
             strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
         return request_data

litellm/llms/base_llm/audio_transcription/transformation.py

@@ -40,6 +40,16 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
     def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]:
         pass
 
+    @property
+    def supports_subtitle_synthesis(self) -> bool:
+        """
+        Opt-in for providers without a native srt/vtt response body: when True
+        and the user asked for response_format srt/vtt, the http handler
+        synthesizes the subtitle document from the word timestamps the
+        provider's TranscriptionResponse carries in `words`.
+        """
+        return False
+
     def get_complete_url(
         self,
         api_base: str | None,

litellm/llms/base_llm/chat/transformation.py

@@ -46,8 +46,10 @@ class BaseLLMException(Exception):
         request: httpx.Request | None = None,
         response: httpx.Response | None = None,
         body: dict | None = None,
+        status_code_is_synthesized: bool = False,
     ):
         self.status_code = status_code
+        self.status_code_is_synthesized = status_code_is_synthesized
         self.message: str = message
         self.headers = headers
         if request:

litellm/llms/base_llm/guardrail_translation/utils.py

@@ -209,9 +209,20 @@ def openai_tool_name(tool: object) -> str | None:
     return flat_name if isinstance(flat_name, str) else None
 
 
+def anthropic_tool_names(tool: object) -> tuple[str, ...]:
+    """Every name a /v1/messages tool dict can act under: the flat Anthropic ``name`` plus
+    ``function.name`` for OpenAI-format tools the bridge forwards verbatim. Allowlist checks
+    must see both, or a decoy flat name could smuggle a disallowed ``function.name`` through."""
+    if not isinstance(tool, dict):
+        return ()
+    function: Final = tool.get("function") if tool.get("type") == "function" else None
+    function_name: Final = function.get("name") if isinstance(function, dict) else None
+    return tuple(name for name in (tool.get("name"), function_name) if isinstance(name, str) and name)
+
+
 def anthropic_tool_name(tool: object) -> str | None:
-    name: Final = tool.get("name") if isinstance(tool, dict) else None
-    return name if isinstance(name, str) else None
+    names: Final = anthropic_tool_names(tool)
+    return names[0] if names else None
 
 
 def merge_returned_tools_into_request_tools(

litellm/llms/bedrock/chat/converse_transformation.py

@@ -1543,6 +1543,7 @@ class AmazonConverseConfig(BaseConfig):
         messages: list[AllMessageValues] | None = None,
         headers: dict | None = None,
         drop_params: bool = False,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> CommonRequestObject:
         ## VALIDATE REQUEST
         """
@@ -1605,6 +1606,16 @@ class AmazonConverseConfig(BaseConfig):
                 if point.get("location") == "tool_config":
                     cache_point = self._build_cache_point_block(point.get("control"), model)
                     bedrock_tools.append(ToolBlock(cachePoint=cache_point))
+                    # Spend attribution credits the gateway only for breakpoints it placed, and
+                    # this is the one place a tool_config point becomes one. The hook that reads
+                    # the configuration cannot record it: whether a cachePoint lands depends on
+                    # this provider and on the request carrying tools, neither of which it sees.
+                    if litellm_params is not None:
+                        from litellm.integrations.anthropic_cache_control_hook import (
+                            AnthropicCacheControlHook,
+                        )
+
+                        AnthropicCacheControlHook.record_gateway_injection(litellm_params, 1)
                     break
 
         bedrock_tool_config: ToolConfigBlock | None = None
@@ -1667,6 +1678,7 @@ class AmazonConverseConfig(BaseConfig):
             messages=messages,
             headers=headers,
             drop_params=litellm_params.get("drop_params") is True,
+            litellm_params=litellm_params,
         )
 
         bedrock_messages: Final = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
@@ -1726,6 +1738,7 @@ class AmazonConverseConfig(BaseConfig):
             messages=messages,
             headers=headers,
             drop_params=litellm_params.get("drop_params") is True,
+            litellm_params=litellm_params,
         )
 
         ## TRANSFORMATION ##

litellm/llms/custom_httpx/llm_http_handler.py

@@ -25,6 +25,10 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
     validated_max_agentic_loops,
 )
 from litellm.litellm_core_utils.asyncify import run_async_function
+from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
+    SUBTITLE_RESPONSE_FORMATS,
+    synthesize_subtitle_document,
+)
 from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
 from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
 from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
@@ -1296,9 +1300,23 @@ class BaseLLMHTTPHandler:
         api_key: str | None,
     ) -> TranscriptionResponse:
         """Shared logic for transforming audio transcription responses."""
-        return provider_config.transform_audio_transcription_response(
+        transformed: Final = provider_config.transform_audio_transcription_response(
             raw_response=response,
         )
+        if not provider_config.supports_subtitle_synthesis:
+            return transformed
+        requested_format: Final = optional_params.get("response_format")
+        if not isinstance(requested_format, str) or requested_format not in SUBTITLE_RESPONSE_FORMATS:
+            return transformed
+        document: Final = synthesize_subtitle_document(
+            words=transformed.get("words"),
+            response_format=requested_format,
+        )
+        if document is not None:
+            transformed.text = document
+        if "words" in transformed:
+            delattr(transformed, "words")
+        return transformed
 
     def audio_transcriptions(
         self,
@@ -5929,11 +5947,13 @@ class BaseLLMHTTPHandler:
             BaseEvalsAPIConfig,
         ],
     ):
-        status_code = getattr(e, "status_code", 500)
+        received_status_code: Final = (
+            e.response.status_code if isinstance(e, httpx.HTTPStatusError) else getattr(e, "status_code", None)
+        )
+        status_code = received_status_code if isinstance(received_status_code, int) else 500
         error_headers = getattr(e, "headers", None)
         if isinstance(e, httpx.HTTPStatusError):
             error_text = e.response.text
-            status_code = e.response.status_code
         else:
             error_text = getattr(e, "text", str(e))
         error_response: Final = getattr(e, "response", None)
@@ -5953,13 +5973,17 @@ class BaseLLMHTTPHandler:
                 status_code=status_code,
                 message=error_text,
                 headers=error_headers,
+                status_code_is_synthesized=not isinstance(received_status_code, int),
             )
 
-        raise provider_config.get_error_class(
+        provider_error: Final = provider_config.get_error_class(
             error_message=error_text,
             status_code=status_code,
             headers=error_headers,
         )
+        if not isinstance(received_status_code, int):
+            provider_error.status_code_is_synthesized = True
+        raise provider_error
 
     @staticmethod
     def _append_query_params(url: str, query_params: RealtimeQueryParams | None) -> str:

litellm/llms/dashscope/image_generation/transformation.py

@@ -11,7 +11,7 @@ Request format:
     "input": {
         "messages": [{"role": "user", "content": [{"text": "<prompt>"}]}]
     },
-    "parameters": {"size": "1024*1024", ...}
+    "parameters": {"size": "1024*1024", "n": 1, ...}
 }
 
 Response format:
@@ -19,7 +19,7 @@ Response format:
     "output": {
         "choices": [{"message": {"content": [{"image": "<url>"}]}}]
     },
-    "usage": {"input_tokens": 0, "output_tokens": 0, "width": 1024, "height": 1024, "image_count": 1}
+    "usage": {"output_width": 1024, "output_height": 1024, "output_image_count": 1}
 }
 """
 
@@ -46,6 +46,8 @@ else:
 
 DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
 
+CHAT_COMPATIBLE_MODE_PATH: Final = "/compatible-mode/v1"
+
 # Maps OpenAI size strings (WxH) to DashScope size strings (W*H)
 OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
     "256x256": "256*256",
@@ -59,7 +61,8 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
 
 class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
     """
-    Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro).
+    Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro,
+    qwen-image-3.0, qwen-image-3.0-pro).
     """
 
     def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
@@ -82,8 +85,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
             if k == "size":
                 # Convert "WxH" → "W*H"
                 mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*"))
-            elif k == "n":
-                mapped["image_count"] = v
+            else:
+                mapped[k] = v
         return mapped
 
     def get_complete_url(
@@ -95,7 +98,10 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
         litellm_params: dict,
         stream: bool | None = None,
     ) -> str:
-        return api_base or get_secret_str("DASHSCOPE_API_BASE_IMAGE") or DEFAULT_API_BASE
+        image_api_base: Final = (
+            api_base if api_base and not api_base.rstrip("/").endswith(CHAT_COMPATIBLE_MODE_PATH) else None
+        )
+        return image_api_base or get_secret_str("DASHSCOPE_API_BASE_IMAGE") or DEFAULT_API_BASE
 
     def validate_environment(
         self,

litellm/llms/gemini/audio_transcription/transformation.py

@@ -4,6 +4,7 @@ from typing import Final
 
 from httpx import Headers, Response
 
+from litellm.litellm_core_utils.audio_utils.subtitle_utils import SUBTITLE_RESPONSE_FORMATS
 from litellm.litellm_core_utils.audio_utils.utils import (
     normalize_transcription_language_to_bcp47,
     process_audio_file,
@@ -48,6 +49,10 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
     ) -> list[OpenAIAudioTranscriptionOptionalParams]:  # mutable-ok: BaseAudioTranscriptionConfig signature
         return ["language", "response_format", "timestamp_granularities"]  # mutable-ok: base contract returns a list
 
+    @property
+    def supports_subtitle_synthesis(self) -> bool:
+        return True
+
     def map_openai_params(
         self,
         non_default_params: Mapping[str, object],
@@ -215,16 +220,17 @@ def _language_config(language: object) -> GeminiTranscriptionConfig:
     return language_config
 
 
-def _timestamp_config(timestamp_granularities: object) -> GeminiTranscriptionConfig:
-    if isinstance(timestamp_granularities, list) and "word" in timestamp_granularities:
-        return _WORD_TIMESTAMP_CONFIG
-    return _EMPTY_TRANSCRIPTION_CONFIG
+def _timestamp_config(timestamp_granularities: object, response_format: object) -> GeminiTranscriptionConfig:
+    wants_word_timestamps: Final = (
+        isinstance(timestamp_granularities, list) and "word" in timestamp_granularities
+    ) or (isinstance(response_format, str) and response_format in SUBTITLE_RESPONSE_FORMATS)
+    return _WORD_TIMESTAMP_CONFIG if wants_word_timestamps else _EMPTY_TRANSCRIPTION_CONFIG
 
 
 def _build_transcription_config(optional_params: Mapping[str, object]) -> GeminiTranscriptionConfig:
     transcription_config: Final[GeminiTranscriptionConfig] = {
         **_language_config(optional_params.get("language")),
-        **_timestamp_config(optional_params.get("timestamp_granularities")),
+        **_timestamp_config(optional_params.get("timestamp_granularities"), optional_params.get("response_format")),
     }
     return transcription_config
 

litellm/llms/gemini/chat/transformation.py

@@ -8,7 +8,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
     convert_url_to_base64,
 )
-from litellm.types.llms.openai import AllMessageValues, ChatCompletionFileObject
+from litellm.types.llms.openai import AllMessageValues, ChatCompletionFileObject, ChatCompletionImageObject
 from litellm.types.llms.vertex_ai import ContentType, PartType
 from litellm.utils import supports_reasoning
 
@@ -16,6 +16,13 @@ from ...vertex_ai.gemini.transformation import _gemini_convert_messages_with_his
 from ...vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
 
 
+def _image_url_fields(img_element: ChatCompletionImageObject) -> tuple[str | None, str | None, str | None]:
+    image_value: Final = img_element.get("image_url")
+    if isinstance(image_value, dict):
+        return image_value.get("url"), image_value.get("format"), image_value.get("detail")
+    return image_value, None, None
+
+
 class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
     """
     Reference: https://ai.google.dev/api/rest/v1beta/GenerationConfig
@@ -118,16 +125,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
                 _parts: list[PartType] = []
                 for element in _message_content:
                     if element.get("type") == "image_url":
-                        img_element = element
-                        _image_url: str | None = None
-                        format: str | None = None
-                        detail: str | None = None
-                        if isinstance(img_element.get("image_url"), dict):
-                            _image_url = img_element["image_url"].get("url")
-                            format = img_element["image_url"].get("format")
-                            detail = img_element["image_url"].get("detail")
-                        else:
-                            _image_url = img_element.get("image_url")
+                        img_element = cast(ChatCompletionImageObject, element)  # cast-ok: runtime type tag checked
+                        _image_url, format, detail = _image_url_fields(img_element)
                         if _image_url and "https://" in _image_url:
                             image_obj = convert_to_anthropic_image_obj(_image_url, format=format)
                             converted_image_url = convert_generic_image_chunk_to_openai_image_obj(image_obj)

litellm/llms/mistral/chat/transformation.py

@@ -292,7 +292,7 @@ class MistralConfig(OpenAIGPTConfig):
                         file_id = file_content.get("file", {}).get("file_id")
                         if file_id:
                             # Replace 'file' with 'file_id'
-                            file_content["file_id"] = file_id
+                            file_content["file_id"] = file_id  # pyright: ignore[reportGeneralTypeIssues]  # legacy in-place rewrite of the block shape
                             file_content.pop("file", None)
         return messages
 

litellm/llms/moonshot/chat/transformation.py

@@ -2,7 +2,7 @@
 Translates from OpenAI's `/v1/chat/completions` to Moonshot AI's `/v1/chat/completions`
 """
 
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from typing import Any, Final, Literal, cast, overload
 
 import litellm
@@ -16,6 +16,15 @@ from litellm.utils import supports_reasoning
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
 
+def _reasoning_effort_string(value: object) -> str | None:
+    """The /v1/messages and /v1/responses bridges wrap the level as {"effort", "summary"} for
+    providers with a reasoning-summary surface. Moonshot's API takes only the bare string and 400s
+    on an object, so the level is unwrapped and the summary, which has no Moonshot equivalent, is
+    dropped."""
+    effort: Final = value.get("effort") if isinstance(value, Mapping) else value
+    return effort if isinstance(effort, str) else None
+
+
 class MoonshotChatConfig(OpenAIGPTConfig):
     @overload
     def _transform_messages(
@@ -93,20 +102,18 @@ class MoonshotChatConfig(OpenAIGPTConfig):
         - functions parameter is not supported (use tools instead)
         - tool_choice doesn't support "required" value
         - kimi-thinking-preview doesn't support tool calls at all
+
+        A reasoning model additionally takes `reasoning_effort`, which the OpenAI base list this
+        subtracts from does not carry, so it has to be added back rather than merely kept.
         """
-        excluded_params: Final[list[str]] = ["functions"]
-
-        # kimi-thinking-preview has additional limitations
-        if "kimi-thinking-preview" in model:
-            excluded_params.extend(["tools", "tool_choice"])
-
+        excluded_params: Final = frozenset(
+            ("functions", "tools", "tool_choice") if "kimi-thinking-preview" in model else ("functions",)
+        )
         base_openai_params: Final = super().get_supported_openai_params(model=model)
-        final_params: Final[list[str]] = []
-        for param in base_openai_params:
-            if param not in excluded_params:
-                final_params.append(param)
-
-        return final_params
+        supported: Final = [param for param in base_openai_params if param not in excluded_params]
+        if supports_reasoning(model=model, custom_llm_provider="moonshot"):
+            return [*supported, "reasoning_effort"]
+        return supported
 
     def map_openai_params(
         self,
@@ -126,7 +133,12 @@ class MoonshotChatConfig(OpenAIGPTConfig):
         for param, value in non_default_params.items():
             if param == "max_completion_tokens":
                 optional_params["max_tokens"] = value
-            elif param in supported_openai_params:
+            elif param not in supported_openai_params:
+                continue
+            elif param == "reasoning_effort":
+                if (effort := _reasoning_effort_string(value)) is not None:
+                    optional_params["reasoning_effort"] = effort
+            else:
                 optional_params[param] = value
 
         ##########################################

litellm/llms/openai/chat/gpt_transformation.py

@@ -18,6 +18,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
     _should_convert_tool_call_to_json_mode,
 )
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    drop_tool_reference_parts_from_tool_messages,
     get_tool_call_names,
     hoist_images_from_tool_messages,
 )
@@ -336,7 +337,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         self, messages: list[AllMessageValues], model: str, is_async: bool = False
     ) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]:
         """OpenAI no longer supports image_url as a string, so we need to convert it to a dict"""
-        hoisted_messages: Final = hoist_images_from_tool_messages(messages)
+        stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(messages)
+        hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
 
         async def _async_transform():
             for message in hoisted_messages:

litellm/llms/soniox/common_utils.py

@@ -4,6 +4,11 @@ Shared utilities for the Soniox provider (https://soniox.com).
 
 from typing import Any, Final
 
+from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
+    SubtitleToken,
+    render_subtitle_tokens_as_srt,
+    render_subtitle_tokens_as_vtt,
+)
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 
 # Soniox API base URL.
@@ -109,121 +114,13 @@ def render_soniox_tokens(tokens: list[dict[str, Any]]) -> str:
     return "".join(text_parts)
 
 
-# ---------------------------------------------------------------------------
-# SRT / VTT subtitle rendering
-# ---------------------------------------------------------------------------
-
-# Maximum number of tokens to group into a single subtitle cue.
-_CUE_MAX_TOKENS: Final[int] = 15
-
-# Maximum duration (in ms) for a single cue before forcing a break.
-_CUE_MAX_DURATION_MS: Final[int] = 5000
-
-
-def _format_timestamp_srt(ms: int) -> str:
-    """Format milliseconds as SRT timestamp: HH:MM:SS,mmm"""
-    ms = max(ms, 0)
-    hours: Final = ms // 3_600_000
-    ms %= 3_600_000
-    minutes: Final = ms // 60_000
-    ms %= 60_000
-    seconds: Final = ms // 1_000
-    millis: Final = ms % 1_000
-    return f"{hours:02d}:{minutes:02d}:{seconds:02d},{millis:03d}"
-
-
-def _format_timestamp_vtt(ms: int) -> str:
-    """Format milliseconds as VTT timestamp: HH:MM:SS.mmm"""
-    ms = max(ms, 0)
-    hours: Final = ms // 3_600_000
-    ms %= 3_600_000
-    minutes: Final = ms // 60_000
-    ms %= 60_000
-    seconds: Final = ms // 1_000
-    millis: Final = ms % 1_000
-    return f"{hours:02d}:{minutes:02d}:{seconds:02d}.{millis:03d}"
-
-
-def _group_tokens_into_cues(
-    tokens: list[dict[str, Any]],
-) -> list[dict[str, Any]]:
-    """
-    Group Soniox tokens into subtitle cues.
-
-    Each cue has:
-      - start_ms: int
-      - end_ms: int
-      - text: str
-
-    Grouping heuristics:
-      - A new cue starts when token count exceeds _CUE_MAX_TOKENS.
-      - A new cue starts when duration exceeds _CUE_MAX_DURATION_MS.
-      - A new cue starts when the speaker changes (if diarization is on).
-      - Tokens without timestamps are appended to the current cue.
-    """
-    cues: Final[list[dict[str, Any]]] = []
-    current_tokens: list[str] = []
-    current_start: int | None = None
-    current_end: int | None = None
-    current_speaker: Any | None = None
-
-    def _flush() -> None:
-        if current_tokens and current_start is not None:
-            text: Final = "".join(current_tokens).strip()
-            if text:
-                cues.append(
-                    {
-                        "start_ms": current_start,
-                        "end_ms": (current_end if current_end is not None else current_start),
-                        "text": text,
-                    }
-                )
-
-    for token in tokens:
-        start_ms = token.get("start_ms")
-        end_ms = token.get("end_ms")
-        text = token.get("text", "")
-        speaker = token.get("speaker")
-
-        # Skip tokens with no timestamp data entirely if we have no cue started
-        if start_ms is None and current_start is None:
-            continue
-
-        # Speaker change forces a new cue
-        if speaker is not None and speaker != current_speaker:
-            _flush()
-            current_tokens = []
-            current_start = start_ms
-            current_end = end_ms
-            current_speaker = speaker
-            current_tokens.append(text)
-            continue
-
-        # Duration or token count exceeded -> flush
-        should_break = False
-        if (
-            len(current_tokens) >= _CUE_MAX_TOKENS
-            or current_start is not None
-            and start_ms is not None
-            and (start_ms - current_start) >= _CUE_MAX_DURATION_MS
-        ):
-            should_break = True
-
-        if should_break:
-            _flush()
-            current_tokens = []
-            current_start = start_ms
-            current_end = end_ms
-            current_tokens.append(text)
-        else:
-            if current_start is None:
-                current_start = start_ms
-            if end_ms is not None:
-                current_end = end_ms
-            current_tokens.append(text)
-
-    _flush()
-    return cues
+def _soniox_token_to_subtitle_token(token: dict[str, Any]) -> SubtitleToken:
+    return SubtitleToken(
+        text=token.get("text", ""),
+        start_ms=token.get("start_ms"),
+        end_ms=token.get("end_ms"),
+        speaker=token.get("speaker"),
+    )
 
 
 def render_soniox_tokens_as_srt(tokens: list[dict[str, Any]]) -> str:
@@ -232,20 +129,7 @@ def render_soniox_tokens_as_srt(tokens: list[dict[str, Any]]) -> str:
 
     Returns an empty string if no tokens have timestamp data.
     """
-    cues: Final = _group_tokens_into_cues(tokens)
-    if not cues:
-        return ""
-
-    lines: Final[list[str]] = []
-    for idx, cue in enumerate(cues, start=1):
-        start = _format_timestamp_srt(cue["start_ms"])
-        end = _format_timestamp_srt(cue["end_ms"])
-        lines.append(str(idx))
-        lines.append(f"{start} --> {end}")
-        lines.append(cue["text"])
-        lines.append("")  # blank line between cues
-
-    return "\n".join(lines)
+    return render_subtitle_tokens_as_srt(tuple(_soniox_token_to_subtitle_token(token) for token in tokens))
 
 
 def render_soniox_tokens_as_vtt(tokens: list[dict[str, Any]]) -> str:
@@ -254,14 +138,4 @@ def render_soniox_tokens_as_vtt(tokens: list[dict[str, Any]]) -> str:
 
     Returns the VTT header even if no cues are present.
     """
-    cues: Final = _group_tokens_into_cues(tokens)
-
-    lines: Final[list[str]] = ["WEBVTT", ""]
-    for cue in cues:
-        start = _format_timestamp_vtt(cue["start_ms"])
-        end = _format_timestamp_vtt(cue["end_ms"])
-        lines.append(f"{start} --> {end}")
-        lines.append(cue["text"])
-        lines.append("")  # blank line between cues
-
-    return "\n".join(lines)
+    return render_subtitle_tokens_as_vtt(tuple(_soniox_token_to_subtitle_token(token) for token in tokens))

litellm/llms/tencent/chat/transformation.py

@@ -3,14 +3,36 @@ Translates from OpenAI's `/v1/chat/completions` to Tencent TokenHub's
 OpenAI-compatible endpoint.
 """
 
-from typing import Final
+from collections.abc import Mapping
+from typing import Final, TypedDict
 
+from typing_extensions import ReadOnly
+
+import litellm
 from litellm.secret_managers.main import get_secret_str
 from litellm.utils import supports_reasoning
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
 
+class ThinkingPayload(TypedDict, total=False):
+    """Tencent TokenHub `thinking` object.
+
+    `type` ("enabled"/"disabled"/"adaptive") is required by TokenHub when the
+    object is passed; `budget_tokens` is auto-filled server-side when omitted.
+    Ref: https://www.tencentcloud.com/document/product/1300/82345
+    """
+
+    type: ReadOnly[str]
+    budget_tokens: ReadOnly[int]
+
+
+class ThinkingExtraBody(TypedDict, total=False):
+    """`extra_body` payload carrying TokenHub's `thinking` object."""
+
+    thinking: ReadOnly[Mapping[str, object]]
+
+
 class TencentChatConfig(OpenAIGPTConfig):
     def get_supported_openai_params(self, model: str) -> list:
         params: Final = super().get_supported_openai_params(model)
@@ -25,18 +47,71 @@ class TencentChatConfig(OpenAIGPTConfig):
         model: str,
         drop_params: bool,
     ) -> dict:
-        optional_params = super().map_openai_params(non_default_params, optional_params, model, drop_params)
+        mapped_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
 
-        thinking_value: Final = optional_params.pop("thinking", None)
-        reasoning_effort: Final = optional_params.pop("reasoning_effort", None)
+        thinking_value: Final = mapped_params.pop("thinking", None)
+        reasoning_effort: Final = mapped_params.pop("reasoning_effort", None)
 
-        if thinking_value is not None:
-            if isinstance(thinking_value, dict):
-                optional_params["thinking"] = thinking_value
-        elif reasoning_effort is not None and reasoning_effort != "none":
-            optional_params["thinking"] = {"type": "enabled"}
+        thinking: Final = self._resolve_thinking_payload(
+            model=model,
+            thinking_value=thinking_value,  # pyright: ignore[reportUnknownArgumentType]  # value popped from the untyped provider params dict
+            reasoning_effort=reasoning_effort,  # pyright: ignore[reportUnknownArgumentType]  # value popped from the untyped provider params dict
+        )
+        if thinking is not None:
+            # TokenHub expects `thinking` in the request JSON body, but the
+            # OpenAI SDK's chat.completions.create() rejects unknown top-level
+            # kwargs, so it travels via `extra_body`, which the SDK merges into
+            # the payload. A plain assignment is merge-safe: get_optional_params
+            # spreads this dict into its own extra_body assembly downstream.
+            extra_body: Final[ThinkingExtraBody] = {"thinking": thinking}
+            mapped_params["extra_body"] = extra_body
+        return mapped_params
 
-        return optional_params
+    @classmethod
+    def _resolve_thinking_payload(
+        cls,
+        model: str,
+        thinking_value: object,
+        reasoning_effort: object,
+    ) -> Mapping[str, object] | None:
+        if isinstance(thinking_value, dict):
+            return cls._coerce_thinking_type_for_model(model=model, thinking=thinking_value)  # pyright: ignore[reportUnknownArgumentType]  # isinstance narrows to dict[Unknown, Unknown] out of the untyped provider params dict
+        if isinstance(reasoning_effort, str):
+            # TokenHub recommends explicitly disabling thinking rather than
+            # relying on per-model defaults (deepseek-v4-* default to enabled).
+            payload: Final[ThinkingPayload] = {"type": "disabled" if reasoning_effort == "none" else "enabled"}
+            return cls._coerce_thinking_type_for_model(model=model, thinking=payload)
+        return None
+
+    @staticmethod
+    def _coerce_thinking_type_for_model(model: str, thinking: Mapping[str, object]) -> Mapping[str, object]:
+        """Coerce `thinking.type` to a value the model accepts.
+
+        MiniMax models on TokenHub only accept "adaptive"/"disabled" and reject
+        "enabled" with a 400; "adaptive" (the model decides when to think) is
+        the closest semantic, so "enabled" is coerced for them. The capability
+        is read from the model map's `supports_adaptive_thinking` flag, so
+        aliases and newly onboarded adaptive-only models need no code change.
+        Ref: https://www.tencentcloud.com/document/product/1300/82345
+        """
+        if thinking.get("type") != "enabled" or not TencentChatConfig._is_adaptive_thinking_model(model):
+            return thinking
+
+        budget: Final[object] = thinking.get("budget_tokens")
+        if isinstance(budget, int):
+            coerced_with_budget: Final[ThinkingPayload] = {"type": "adaptive", "budget_tokens": budget}
+            return coerced_with_budget
+        coerced: Final[ThinkingPayload] = {"type": "adaptive"}
+        return coerced
+
+    @staticmethod
+    def _is_adaptive_thinking_model(model: str) -> bool:
+        """Read `supports_adaptive_thinking` from the model map under tencent."""
+        try:
+            model_info: Final[Mapping[str, object]] = litellm.get_model_info(model=model, custom_llm_provider="tencent")
+        except Exception:  # noqa: BLE001  # get_model_info raises a bare Exception for unmapped models
+            return False
+        return model_info.get("supports_adaptive_thinking") is True
 
     def _get_openai_compatible_provider_info(
         self, api_base: str | None, api_key: str | None

litellm/llms/together_ai/chat/transformation.py

@@ -4,7 +4,8 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
 Docs: https://docs.together.ai/docs/chat-overview
 """
 
-from collections.abc import Callable, Container, Coroutine
+from collections.abc import Callable, Container, Coroutine, Mapping
+from types import MappingProxyType
 from typing import (
     Final,
     Literal,
@@ -12,11 +13,14 @@ from typing import (
     overload,
 )
 
+from typing_extensions import ReadOnly, TypedDict
+
 import litellm
 from litellm._logging import verbose_logger
 from litellm.exceptions import UnsupportedParamsError
+from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
 from litellm.types.llms.openai import AllMessageValues
-from litellm.utils import supports_function_calling, supports_response_schema
+from litellm.utils import supports_function_calling, supports_reasoning, supports_response_schema
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
@@ -38,6 +42,34 @@ def _registry_verdict(model: str, flag: str, check: Callable[[str], bool]) -> bo
     return None
 
 
+ADJUSTABLE_EFFORT_REASONING_MODELS: Final = frozenset(
+    {
+        "openai/gpt-oss-120b",
+        "openai/gpt-oss-20b",
+    }
+)
+HYBRID_REASONING_MODELS: Final = frozenset(
+    {
+        "MiniMaxAI/MiniMax-M3",
+        "Qwen/Qwen3.5-9B",
+        "Qwen/Qwen3.6-Plus",
+        "deepseek-ai/DeepSeek-V4-Pro",
+        "moonshotai/Kimi-K3",
+        "nvidia/nemotron-3-ultra-550b-a55b",
+        "zai-org/GLM-5.2",
+    }
+)
+HIGH_MAX_EFFORT_MODEL_PREFIX: Final = "deepseek-ai/DeepSeek-V4-Pro"
+EFFORT_TRANSLATION: Final = MappingProxyType({"minimal": "low", "xhigh": "high", "max": "high"})
+HIGH_MAX_EFFORT_TRANSLATION: Final = MappingProxyType(
+    {"minimal": "high", "low": "high", "medium": "high", "xhigh": "max"}
+)
+
+
+class TogetherReasoningToggle(TypedDict):
+    enabled: ReadOnly[bool]
+
+
 def _function_calling_verdict(model: str) -> bool | None:
     return _registry_verdict(
         model,
@@ -83,6 +115,38 @@ def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params:
     )
 
 
+def _supports_together_reasoning(model: str) -> bool:
+    if model in ADJUSTABLE_EFFORT_REASONING_MODELS or model in HYBRID_REASONING_MODELS:
+        return True
+    if model.startswith(HIGH_MAX_EFFORT_MODEL_PREFIX):
+        return True
+    return supports_reasoning(model, custom_llm_provider="together_ai")
+
+
+def _adjustable_effort(effort: str, model: str) -> str:
+    if effort == "none":
+        verbose_logger.debug(
+            "together_ai model %s cannot disable reasoning; mapping reasoning_effort=none to low", model
+        )
+        return "low"
+    return EFFORT_TRANSLATION.get(effort, effort)
+
+
+def _reasoning_effort_payload(effort: str, model: str) -> Mapping[str, object]:
+    if effort == "default":
+        return MappingProxyType({})
+    if model in ADJUSTABLE_EFFORT_REASONING_MODELS:
+        return MappingProxyType({"reasoning_effort": _adjustable_effort(effort, model)})
+    if effort == "none":
+        disable_reasoning: Final[TogetherReasoningToggle] = {"enabled": False}
+        return MappingProxyType({"reasoning": disable_reasoning})
+    if effort in (declared_reasoning_efforts_for_model(model, "together_ai") or ()):
+        return MappingProxyType({"reasoning_effort": effort})
+    if model.startswith(HIGH_MAX_EFFORT_MODEL_PREFIX):
+        return MappingProxyType({"reasoning_effort": HIGH_MAX_EFFORT_TRANSLATION.get(effort, effort)})
+    return MappingProxyType({"reasoning_effort": EFFORT_TRANSLATION.get(effort, effort)})
+
+
 def _drop_response_format(passed_params: Container[str], model: str, drop_params: bool) -> bool:
     if "response_format" not in passed_params:
         return False
@@ -153,6 +217,15 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
             return super()._transform_messages(stripped, model, is_async=True)
         return super()._transform_messages(stripped, model, is_async=False)
 
+    def get_supported_openai_params(self, model: str) -> list:  # mutable-ok: inherited contract
+        supported_params: Final = super().get_supported_openai_params(model)
+        if not _supports_together_reasoning(model):
+            return supported_params
+        return [  # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
+            *supported_params,
+            "reasoning_effort",
+        ]
+
     def map_openai_params(
         self,
         non_default_params: dict,
@@ -165,4 +238,10 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
             mapped_openai_params.pop(param)
         if _drop_response_format(mapped_openai_params, model, drop_params):
             mapped_openai_params.pop("response_format")
+        effort: Final = mapped_openai_params.get("reasoning_effort")
+        if not isinstance(effort, str):
+            return mapped_openai_params
+        mapped_openai_params.pop("reasoning_effort")
+        for key, value in _reasoning_effort_payload(effort, model).items():
+            mapped_openai_params.setdefault(key, value)
         return mapped_openai_params

litellm/llms/together_ai/cost_calculator.py

@@ -3,6 +3,7 @@ Handles calculating cost for together ai models
 """
 
 import re
+from collections.abc import Mapping
 from typing import Final
 
 from litellm.constants import (
@@ -18,6 +19,12 @@ from litellm.constants import (
 from litellm.types.utils import CallTypes
 
 
+def has_together_registry_pricing(model: str, cost_map: Mapping[str, object]) -> bool:
+    stripped: Final = model.removeprefix("together_ai/")
+    entry: Final = cost_map.get(f"together_ai/{stripped}")
+    return isinstance(entry, Mapping) and "input_cost_per_token" in entry
+
+
 # Extract the number of billion parameters from the model name
 # only used for together_computer LLMs
 def get_model_params_and_category(model_name, call_type: CallTypes) -> str:

litellm/main.py

@@ -531,6 +531,7 @@ async def acompletion(
             tools=tools,
             prompt_label=kwargs.get("prompt_label", None),
             prompt_version=kwargs.get("prompt_version", None),
+            request_kwargs=kwargs,
         )
         #########################################################
         # if the chat completion logging hook removed all tools,
@@ -5246,6 +5247,7 @@ def completion(
             prompt_variables=prompt_variables,
             prompt_label=kwargs.get("prompt_label", None),
             prompt_version=kwargs.get("prompt_version", None),
+            request_kwargs=kwargs,
         )
 
     ### LITELLM SYSTEM PROMPT ###

litellm/model_prices_and_context_window_backup.json

@@ -9315,6 +9315,11 @@
         "max_tokens": 131072,
         "mode": "chat",
         "output_cost_per_token": 1.65e-05,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
         "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k3-through-fireworks-ai-on-microsoft-foundry/4540187",
         "supported_modalities": [
             "text",
@@ -14647,6 +14652,22 @@
             "/v1/images/generations"
         ]
     },
+    "dashscope/qwen-image-3.0": {
+        "litellm_provider": "dashscope",
+        "mode": "image_generation",
+        "source": "https://www.alibabacloud.com/help/en/model-studio/models",
+        "supported_endpoints": [
+            "/v1/images/generations"
+        ]
+    },
+    "dashscope/qwen-image-3.0-pro": {
+        "litellm_provider": "dashscope",
+        "mode": "image_generation",
+        "source": "https://www.alibabacloud.com/help/en/model-studio/models",
+        "supported_endpoints": [
+            "/v1/images/generations"
+        ]
+    },
     "databricks/databricks-bge-large-en": {
         "cache_creation_input_token_cost": 1.0003e-07,
         "cache_read_input_token_cost": 1.0003e-07,
@@ -31897,6 +31918,11 @@
         "max_tokens": 1048576,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
         "source": "https://platform.kimi.ai/docs/pricing/chat-k3",
         "supports_function_calling": true,
         "supports_reasoning": true,
@@ -36505,6 +36531,14 @@
         "litellm_provider": "perplexity",
         "mode": "responses",
         "output_cost_per_token": 1.5e-05,
+        "reasoning_effort_levels": [
+            "minimal",
+            "low",
+            "medium",
+            "high",
+            "xhigh",
+            "max"
+        ],
         "source": "https://docs.perplexity.ai/docs/agent-api/models",
         "supports_web_search": true,
         "supports_reasoning": true,
@@ -38807,14 +38841,14 @@
         "supports_reasoning": true
     },
     "together_ai/Qwen/Qwen3.7-Max": {
-        "cache_read_input_token_cost": 1.3e-07,
-        "input_cost_per_token": 1.25e-06,
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 2.5e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 1000000,
         "max_output_tokens": 1000000,
         "max_tokens": 1000000,
         "mode": "chat",
-        "output_cost_per_token": 3.75e-06,
+        "output_cost_per_token": 7.5e-06,
         "source": "https://docs.together.ai/docs/serverless-models",
         "supports_prompt_caching": true
     },
@@ -38986,6 +39020,11 @@
         "max_tokens": 1048576,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
         "source": "https://docs.together.ai/docs/serverless-models",
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
@@ -39069,6 +39108,24 @@
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
+    "together_ai/zai-org/GLM-5.3-Flash": {
+        "cache_read_input_token_cost": 3e-08,
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048575,
+        "max_output_tokens": 1048575,
+        "max_tokens": 1048575,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "tts-1": {
         "input_cost_per_character": 1.5e-05,
         "litellm_provider": "openai",
@@ -51061,6 +51118,26 @@
         "supports_reasoning": true,
         "supports_vision": false
     },
+    "tencent/minimax-m3": {
+        "cache_creation_input_token_cost": 0.0,
+        "cache_read_input_token_cost": 6e-08,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_cache_hit": 6e-08,
+        "litellm_provider": "tencent",
+        "max_input_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://www.tencentcloud.com/products/tokenhub",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_adaptive_thinking": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_vision": false
+    },
     "cognition/swe-1.6": {
         "input_cost_per_token": 5e-07,
         "output_cost_per_token": 2.5e-06,
@@ -51757,6 +51834,11 @@
         "max_tokens": 131072,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
         "supports_reasoning": true,
@@ -51821,6 +51903,11 @@
         "max_tokens": 131072,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
         "supports_reasoning": true,
@@ -51837,6 +51924,11 @@
         "max_tokens": 131072,
         "mode": "chat",
         "output_cost_per_token": 2.25e-05,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
         "supports_reasoning": true,
@@ -51853,6 +51945,11 @@
         "max_tokens": 131072,
         "mode": "chat",
         "output_cost_per_token": 1.65e-05,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
         "supports_reasoning": true,
@@ -52025,6 +52122,11 @@
         "max_tokens": 131072,
         "mode": "chat",
         "output_cost_per_token": 2.25e-05,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
         "supports_reasoning": true,
@@ -52041,6 +52143,11 @@
         "max_tokens": 131072,
         "mode": "chat",
         "output_cost_per_token": 1.65e-05,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
         "source": "https://docs.fireworks.ai/serverless/pricing",
         "supports_function_calling": true,
         "supports_reasoning": true,

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -34,6 +34,7 @@ from mcp.types import (
 )
 from mcp.types import Tool as MCPTool
 from pydantic import AnyUrl, BaseModel
+from typing_extensions import ReadOnly
 
 import litellm
 from litellm._logging import verbose_logger
@@ -72,6 +73,7 @@ from litellm.proxy._experimental.mcp_server.oauth2_token_cache import (
     MCPPerUserTokenCache,
     mcp_per_user_token_cache,
     resolve_mcp_auth,
+    resolved_token_header,
 )
 from litellm.proxy._experimental.mcp_server.oauth_utils import (
     _redact_mcp_resource_url,
@@ -99,6 +101,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchange_
     build_token_exchanger,
 )
 from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
+    DEFAULT_CREDENTIAL_HEADER,
     AuthorizationCodeConfig,
     ClientCredentialsConfig,
     CredError,
@@ -153,6 +156,8 @@ from litellm.types.mcp import (
     MCPAuth,
     MCPStdioConfig,
     MCPTokenEndpointAuthMethod,
+    has_header,
+    without_header,
 )
 from litellm.types.mcp_server.mcp_server_manager import (
     MCPInfo,
@@ -349,6 +354,7 @@ class MCPServerConfig(TypedDict, total=False):
     audience: str
     subject_token_type: str
     upstream_resource: str
+    upstream_token_header: ReadOnly[str]
     id_jag_resource_token_endpoint: str
     id_jag_resource: str
     client_private_key: str
@@ -828,18 +834,6 @@ def _should_strip_caller_authorization(
     )
 
 
-def _without_authorization(
-    headers: dict[str, str] | None,
-) -> dict[str, str] | None:
-    """A copy of ``headers`` with any ``Authorization`` key removed (case-insensitive), or
-    None if nothing remains. Drops only the credential, keeping other forwarded headers.
-    """
-    if not headers:
-        return None
-    filtered: Final = {k: v for k, v in headers.items() if k.lower() != "authorization"}
-    return filtered or None
-
-
 def _format_byok_openapi_auth_header(mcp_server: MCPServer, mcp_auth_header: str) -> str:
     """Format a raw BYOK credential for OpenAPI tool ``Authorization`` injection.
 
@@ -914,7 +908,9 @@ def _resolve_openapi_tool_auth(
 
     if isinstance(per_server, dict):
         authorization: Final = next((v for k, v in per_server.items() if k.lower() == "authorization"), None)
-        merged: Final = merge_mcp_headers(extra_headers=forwarded, static_headers=_without_authorization(per_server))
+        merged: Final = merge_mcp_headers(
+            extra_headers=forwarded, static_headers=without_header(per_server, DEFAULT_CREDENTIAL_HEADER)
+        )
         if authorization is None:
             byok: Final = _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None
             return byok, merged, mcp_auth_header
@@ -981,7 +977,7 @@ def _client_forwarded_authorization_headers(
         raw_headers=raw_headers,
         user_api_key_auth=user_api_key_auth,
     ):
-        return _without_authorization(extra_headers)
+        return without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER)
     return extra_headers
 
 
@@ -994,7 +990,7 @@ def _take_forwarded_authorization(
     if not headers:
         return None, headers
     value: Final = next((v for k, v in headers.items() if k.lower() == "authorization"), None)
-    return value, _without_authorization(headers)
+    return value, without_header(headers, DEFAULT_CREDENTIAL_HEADER)
 
 
 def _passthrough_token_from_mcp_auth_header(
@@ -2166,6 +2162,7 @@ class MCPServerManager:
                     DEFAULT_SUBJECT_TOKEN_TYPE,
                 ),
                 upstream_resource=server_config.get("upstream_resource", None),
+                upstream_token_header=server_config.get("upstream_token_header", None),
                 # ID-JAG fields
                 id_jag_resource_token_endpoint=server_config.get("id_jag_resource_token_endpoint", None),
                 id_jag_resource=server_config.get("id_jag_resource", None),
@@ -2698,6 +2695,7 @@ class MCPServerManager:
             or (credentials_dict.get("subject_token_type") if credentials_dict else None)
             or DEFAULT_SUBJECT_TOKEN_TYPE,
             upstream_resource=(credentials_dict.get("upstream_resource") if credentials_dict else None),
+            upstream_token_header=(credentials_dict.get("upstream_token_header") if credentials_dict else None),
             # ID-JAG fields — read from credentials JSON blob
             id_jag_resource_token_endpoint=(
                 credentials_dict.get("id_jag_resource_token_endpoint") if credentials_dict else None
@@ -3525,10 +3523,9 @@ class MCPServerManager:
             case Ok(auth):
                 # NoOpAuth has no header_name and so never conflicts.
                 header_name: Final[str | None] = getattr(auth, "header_name", None)
-                conflicts: Final = bool(
-                    header_name and extra_headers and any(key.lower() == header_name.lower() for key in extra_headers)
-                )
-                if not conflicts:
+                if header_name is None or not extra_headers:
+                    return auth, extra_headers
+                if not has_header(extra_headers, header_name):
                     return auth, extra_headers
                 if isinstance(
                     spec.config,
@@ -3540,9 +3537,10 @@ class MCPServerManager:
                     # guardrail such as MCPJWTSigner, static_headers, or any other injected
                     # Authorization must NOT shadow it (otherwise the upstream gets e.g. the
                     # signer's JWT instead of the minted token and rejects it, and for M2M the
-                    # one-shot 401 refetch is lost with it). Drop the conflicting header so the
-                    # resolved token reaches upstream.
-                    return auth, _without_authorization(extra_headers)
+                    # one-shot 401 refetch is lost with it). Drop only the header the resolved
+                    # credential is about to occupy, so a static credential the operator aimed at a
+                    # DIFFERENT header still reaches upstream.
+                    return auth, without_header(extra_headers, header_name)
                 # Other modes: an Authorization already supplied via extra_headers (a forwarded caller
                 # header or static_headers) is intentional and wins; v1 applies those last.
                 return None, extra_headers
@@ -3650,6 +3648,7 @@ class MCPServerManager:
         ):
             spec = None
         auth_value: Final = await resolve_mcp_auth(resolved_server, mcp_auth_header) if spec is None else None
+        auth_header_name: Final = resolved_token_header(resolved_server, mcp_auth_header) if spec is None else None
 
         # Create sampling and elicitation callbacks for this client
         sampling_cb = (
@@ -3758,6 +3757,7 @@ class MCPServerManager:
                 transport_type=transport,
                 auth_type=resolved_server.auth_type,
                 auth_value=auth_value,
+                auth_header_name=auth_header_name,
                 timeout=(resolved_server.timeout if resolved_server.timeout is not None else MCP_CLIENT_TIMEOUT),
                 extra_headers=extra_headers,
                 aws_auth=aws_auth,
@@ -5306,7 +5306,7 @@ class MCPServerManager:
                     raw_headers=raw_headers,
                     user_api_key_auth=user_api_key_auth,
                 ):
-                    extra_headers = _without_authorization(extra_headers)
+                    extra_headers = without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER)
         elif mcp_server.is_client_forwarded_token:
             extra_headers = _client_forwarded_authorization_headers(
                 mcp_server=mcp_server,

litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py

@@ -7,6 +7,7 @@ with ``client_id``, ``client_secret``, and ``token_url``.
 
 import asyncio
 import hashlib
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Final
 
 import httpx
@@ -313,9 +314,26 @@ async def resolve_mcp_auth(
     1. ``mcp_auth_header`` — per-request/per-user override
     2. OAuth2 client_credentials token — auto-fetched and cached
     3. ``server.authentication_token`` — static token from config/DB
+
+    ``resolved_token_header`` answers, for the same two inputs, which header the value belongs in.
     """
     if mcp_auth_header:
         return mcp_auth_header
     if server.has_client_credentials:
         return await mcp_oauth2_token_cache.async_get_token(server)
     return server.authentication_token
+
+
+def resolved_token_header(
+    server: "MCPServer",
+    mcp_auth_header: str | Mapping[str, str] | None = None,
+) -> str | None:
+    """Which upstream header the value ``resolve_mcp_auth`` just returned belongs in.
+
+    ``None`` means keep the auth_type default. A caller-supplied ``mcp_auth_header`` is the caller's
+    own credential aimed at the slot the upstream normally uses, so it never moves; only the values
+    the gateway resolved from its own config (the minted M2M token, the static token) follow
+    ``upstream_token_header``. Same inputs and same branch order as ``resolve_mcp_auth``, so the two
+    cannot disagree about which case they are in.
+    """
+    return None if mcp_auth_header else server.upstream_token_header

litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py

@@ -47,12 +47,14 @@ def sanitize_openapi_tool_name(raw_name: str) -> str:
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.url_utils import async_safe_get
 from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
     get_async_httpx_client,
     httpxSpecialProvider,
 )
 from litellm.proxy._experimental.mcp_server.tool_registry import (
     global_mcp_tool_registry,
 )
+from litellm.types.mcp import credential_redirect_hook, custom_credential_slot
 
 
 class _OpenAPIJSONSchema(TypedDict, total=False):
@@ -119,6 +121,10 @@ _request_resolved_auth_headers: Final[contextvars.ContextVar[dict[str, str] | No
     "_request_resolved_auth_headers", default=None
 )
 
+_request_upstream_url: Final[contextvars.ContextVar[str | None]] = contextvars.ContextVar(
+    "_request_upstream_url", default=None
+)
+
 
 def _sanitize_path_parameter_value(param_value: object, param_name: str) -> str:
     """Ensure path params cannot introduce directory traversal."""
@@ -349,6 +355,35 @@ def build_input_schema(operation: _OpenAPIOperation) -> dict[str, object]:
     }
 
 
+async def _drop_credential_across_origin(request: httpx.Request) -> None:
+    """Apply this request's cross-origin credential guard, if it needs one.
+
+    Reads the per-request context rather than closing over it so the hook is one stable object, which
+    keeps the guarded client cacheable. A closure would key a new entry per call, and the handler it
+    built would never be closed.
+    """
+    guard: Final = credential_redirect_hook(
+        _request_upstream_url.get() or "", custom_credential_slot(_request_resolved_auth_headers.get())
+    )
+    if guard is not None:
+        await guard(request)
+
+
+def _upstream_client() -> AsyncHTTPHandler:
+    """The HTTP client for one upstream call, guarded when a credential rides a custom slot.
+
+    A resolved credential outside ``Authorization`` is not stripped across origins by the client
+    itself, so this arm installs the same hook the MCP client uses. Both variants come from the
+    shared cache, so a guarded call reuses its connection pool like any other.
+    """
+    if custom_credential_slot(_request_resolved_auth_headers.get()) is None:
+        return get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
+    return get_async_httpx_client(
+        llm_provider=httpxSpecialProvider.MCP,
+        params={"event_hooks": {"request": [_drop_credential_across_origin]}},
+    )
+
+
 def _merge_openapi_tool_request_headers(
     static_headers: dict[str, str],
 ) -> dict[str, str]:
@@ -510,8 +545,9 @@ def create_tool_function(
                 except (json.JSONDecodeError, TypeError):
                     json_body = {"data": body_value}
 
-        client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
+        client: Final = _upstream_client()
         upstream: Final = server_label or f"{original_method.upper()} {path}"
+        url_token: Final = _request_upstream_url.set(url)
 
         try:
             if original_method == "get":
@@ -529,6 +565,8 @@ def create_tool_function(
         except MaskedHTTPStatusError as e:
             _raise_for_upstream_failure(e.response, upstream, relays_upstream_auth)
             raise
+        finally:
+            _request_upstream_url.reset(url_token)
 
         _raise_for_upstream_failure(response, upstream, relays_upstream_auth)
         return response.text

litellm/proxy/_experimental/mcp_server/outbound_credentials/__init__.py

@@ -21,6 +21,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
     Result,
 )
 from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
+    DEFAULT_CREDENTIAL_HEADER,
     Ambient,
     ApiKeyConfig,
     ApiKeySource,
@@ -35,6 +36,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
     ClientCredentialsConfig,
     ClientSecretAuth,
     CredError,
+    HeaderCarrier,
     IdJagConfig,
     NoneConfig,
     PassthroughConfig,
@@ -45,9 +47,11 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
     Subject,
     TokenExchangeConfig,
     parse_auth_spec_kind,
+    validate_header_name,
 )
 
 __all__ = [
+    "DEFAULT_CREDENTIAL_HEADER",
     "Ambient",
     "ApiKeyConfig",
     "ApiKeySource",
@@ -63,6 +67,7 @@ __all__ = [
     "ClientSecretAuth",
     "CredError",
     "Error",
+    "HeaderCarrier",
     "IdJagConfig",
     "NoOpAuth",
     "NoneConfig",
@@ -78,4 +83,5 @@ __all__ = [
     "TokenExchangeConfig",
     "UpstreamCredentialProvider",
     "parse_auth_spec_kind",
+    "validate_header_name",
 ]

litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py

@@ -20,6 +20,7 @@ from typing_extensions import assert_never
 
 from litellm.proxy._experimental.mcp_server.oauth_utils import resolve_upstream_resource
 from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
+    DEFAULT_CREDENTIAL_HEADER,
     ApiKeyConfig,
     AuthorizationCodeConfig,
     ClientAuth,
@@ -45,6 +46,15 @@ _TOKEN_EXCHANGE_SUBJECT_TOKEN_DEFAULT: Final = "urn:ietf:params:oauth:token-type
 _ID_JAG_SUBJECT_TOKEN_DEFAULT: Final = "urn:ietf:params:oauth:token-type:id_token"
 
 
+def token_header(server: MCPServer) -> str:
+    """The upstream header this server's resolved credential occupies.
+
+    One owner for every arm, so no spec builder spells the default itself and a server can never
+    hand two arms different answers.
+    """
+    return server.upstream_token_header or DEFAULT_CREDENTIAL_HEADER
+
+
 def to_subject(user_api_key_auth: UserAPIKeyAuth | None, subject_token: str | None) -> Subject:
     """Map v1's authenticated principal onto the resolver's Subject.
 
@@ -122,7 +132,7 @@ def _oauth2_spec(server: MCPServer, resource: str) -> ServerSpec | None:
         return ServerSpec(
             server_id=server.server_id,
             resource=resource,
-            config=AuthorizationCodeConfig(),
+            config=AuthorizationCodeConfig(header_name=token_header(server)),
         )
     return None
 
@@ -140,6 +150,7 @@ def _client_credentials_spec(server: MCPServer, resource: str) -> ServerSpec:
         server_id=server.server_id,
         resource=resource,
         config=ClientCredentialsConfig(
+            header_name=token_header(server),
             client_id=server.client_id,
             client_secret=SecretStr(server.client_secret) if server.client_secret else None,
             token_url=server.effective_token_url,
@@ -173,6 +184,7 @@ def _token_exchange_spec(server: MCPServer, resource: str) -> ServerSpec | None:
         server_id=server.server_id,
         resource=resource,
         config=TokenExchangeConfig(
+            header_name=token_header(server),
             profile=profile,
             subject_token_type=server.subject_token_type or DEFAULT_SUBJECT_TOKEN_TYPE,
             token_exchange_endpoint=endpoint,
@@ -206,7 +218,7 @@ def _shared_key_spec(
         server_id=server.server_id,
         resource=resource,
         config=ApiKeyConfig(
-            header_name=header_name,
+            header_name=server.upstream_token_header or header_name,
             value_prefix=value_prefix,
             key_source=SharedKey(value=SecretStr(value)),
         ),
@@ -231,6 +243,7 @@ def _id_jag_spec(server: MCPServer, resource: str) -> ServerSpec | None:
         server_id=server.server_id,
         resource=resource,
         config=IdJagConfig(
+            header_name=token_header(server),
             org_token_endpoint=org_token_endpoint,
             resource_token_endpoint=resource_token_endpoint,
             client_id=client_id,

litellm/proxy/_experimental/mcp_server/outbound_credentials/client_credentials.py

@@ -50,6 +50,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
 from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
     ClientCredentialsConfig,
     CredError,
+    HeaderCarrier,
 )
 
 
@@ -328,14 +329,21 @@ class ClientCredentialsBearerAuth(httpx.Auth):
     refetch fails, or the retried request 401s again, the upstream's response stands.
     """
 
-    def __init__(self, access_token: str, refetch: Callable[[str], Awaitable[str | None]]) -> None:
-        self.header_name = "Authorization"
+    def __init__(
+        self,
+        access_token: str,
+        refetch: Callable[[str], Awaitable[str | None]],
+        carrier: HeaderCarrier,
+    ) -> None:
+        self._carrier = carrier
+        self.header_name = carrier.header_name
         self._access_token = SecretStr(access_token)
         self._refetch = refetch
 
     async def async_auth_flow(self, request: httpx.Request) -> AsyncGenerator[httpx.Request, httpx.Response]:
         token: Final = self._access_token.get_secret_value()
-        request.headers[self.header_name] = f"Bearer {token}"
+        name, value = self._carrier.header(token)
+        request.headers[name] = value
         response: Final = yield request
         if response.status_code != 401:
             return
@@ -343,7 +351,8 @@ class ClientCredentialsBearerAuth(httpx.Auth):
         if fresh is None:
             return
         self._access_token = SecretStr(fresh)
-        request.headers[self.header_name] = f"Bearer {fresh}"
+        fresh_name, fresh_value = self._carrier.header(fresh)
+        request.headers[fresh_name] = fresh_value
         yield request
 
     def sync_auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]:

litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py

@@ -145,8 +145,8 @@ class UpstreamCredentialProvider:
                 return await self._token_exchange(subject, server, config)
             case IdJagConfig() as config:
                 return await self._id_jag(subject, server, config)
-            case AuthorizationCodeConfig():
-                return await self._authorization_code(subject, server)
+            case AuthorizationCodeConfig() as config:
+                return await self._authorization_code(subject, server, config)
             case AwsSigV4Config():
                 return _not_implemented(AuthSpecKind.aws_sigv4)
         assert_never(server.config)
@@ -284,15 +284,19 @@ class UpstreamCredentialProvider:
 
         match await self._exchanged_tokens.get_or_compute(slot, _exchange, fingerprint=fingerprint):
             case Ok(access_token):
-                return Ok(StaticHeaderAuth(f"Bearer {access_token}"))
+                header_name, header_value = config.header(access_token)
+                return Ok(StaticHeaderAuth(header_value, header_name=header_name))
             case Error(err):
                 return Error(err)
 
-    async def _authorization_code(self, subject: Subject, server: ServerSpec) -> Result[StaticHeaderAuth, CredError]:
+    async def _authorization_code(
+        self, subject: Subject, server: ServerSpec, config: AuthorizationCodeConfig
+    ) -> Result[StaticHeaderAuth, CredError]:
         token: Final = await self._authz_token(subject, server)
         if token is None:
             return Error(CredError.of_unauthorized("Authorization required: complete the OAuth flow for this server."))
-        return Ok(StaticHeaderAuth(f"Bearer {token.access_token}", header_name="Authorization"))
+        header_name, header_value = config.header(token.access_token)
+        return Ok(StaticHeaderAuth(header_value, header_name=header_name))
 
     async def _client_credentials(
         self, server_id: str, config: ClientCredentialsConfig
@@ -307,7 +311,7 @@ class UpstreamCredentialProvider:
         match await self._client_credentials_source.get(server_id, config):
             case Ok(token):
                 refetch: Final = partial(self._client_credentials_source.refetch, server_id, config)
-                return Ok(ClientCredentialsBearerAuth(token.access_token, refetch))
+                return Ok(ClientCredentialsBearerAuth(token.access_token, refetch, config))
             case Error(err):
                 return Error(err)
 
@@ -332,7 +336,8 @@ class UpstreamCredentialProvider:
             inbound.get_secret_value(), server, config, tenant_id=subject.tenant_id
         ):
             case Ok(token):
-                return Ok(StaticHeaderAuth(f"Bearer {token.access_token}", header_name="Authorization"))
+                header_name, header_value = config.header(token.access_token)
+                return Ok(StaticHeaderAuth(header_value, header_name=header_name))
             case Error(err):
                 return Error(err)
 

litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py

@@ -31,7 +31,7 @@ from enum import Enum
 from typing import Annotated, Final, Literal
 
 from expression import case, tag, tagged_union
-from pydantic import BaseModel, ConfigDict, Field, SecretStr
+from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator
 from typing_extensions import assert_never
 
 from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
@@ -39,7 +39,11 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
     Ok,
     Result,
 )
-from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE
+from litellm.types.mcp import (
+    DEFAULT_CREDENTIAL_HEADER,
+    DEFAULT_SUBJECT_TOKEN_TYPE,
+    normalize_upstream_header_name,
+)
 
 
 class AuthSpecKind(str, Enum):
@@ -161,7 +165,52 @@ class CredError:
         assert_never(self.tag)
 
 
-class AuthorizationCodeConfig(BaseModel):
+def validate_header_name(raw: str) -> Result[str, CredError]:
+    """``normalize_upstream_header_name`` with this package's error-as-value policy.
+
+    The grammar itself lives in ``litellm.types.mcp`` so the v1 model, the management endpoint and
+    this vocabulary all judge a header name the same way while each keeps its own failure shape.
+    """
+    normalized: Final = normalize_upstream_header_name(raw)
+    if normalized is None:
+        return Error(CredError.of_misconfigured(f"invalid upstream header name: {raw!r}"))
+    return Ok(normalized)
+
+
+class HeaderCarrier(BaseModel):
+    """Where a resolved credential is written upstream, and how its value is formatted.
+
+    ``Authorization: Bearer`` is only OAuth's *default* conveyance (RFC 6750 section 2.1), not its
+    only one: an ESB or API gateway commonly terminates its own credential in a private header while
+    a second credential passes through to the origin, so a credential has to be able to say which
+    slot it owns. Modeled like OpenAPI's apiKey scheme, so any upstream convention is expressible
+    (Authorization + Bearer, a raw value on X-API-Key, Ocp-Apim-Subscription-Key, esb-oauth, ...).
+
+    Every config whose credential the gateway mints or holds inherits this, so no resolver arm names
+    a header itself and the conflict rule in ``_resolve_v2_auth`` can always ask the auth object
+    which slot it is about to occupy. ``passthrough`` deliberately does not: it forwards the
+    caller's own credential into the slot the caller used, and mints nothing to place.
+    """
+
+    model_config = ConfigDict(frozen=True)
+    header_name: str = DEFAULT_CREDENTIAL_HEADER
+    value_prefix: str = "Bearer"
+
+    @field_validator("header_name")
+    @classmethod
+    def _check_header_name(cls, value: str) -> str:
+        match validate_header_name(value):
+            case Ok(name):
+                return name
+            case Error(err):
+                raise ValueError(err.summary)
+
+    def header(self, value: str) -> tuple[str, str]:
+        formatted: Final = f"{self.value_prefix} {value}" if self.value_prefix else value
+        return self.header_name, formatted
+
+
+class AuthorizationCodeConfig(HeaderCarrier):
     """Per-user 3LO; the gateway is the OAuth client and stores the user's token.
 
     Endpoints are discovered (RFC 9728 -> RFC 8414) and the client is registered via DCR
@@ -179,7 +228,7 @@ class AuthorizationCodeConfig(BaseModel):
     token_url: str | None = None
 
 
-class ClientCredentialsConfig(BaseModel):
+class ClientCredentialsConfig(HeaderCarrier):
     """M2M service account; one upstream identity for every user.
 
     Fields are optional so the config can be built incomplete: a value may be supplied at
@@ -203,7 +252,7 @@ class ClientCredentialsConfig(BaseModel):
     token_endpoint_auth_method: Literal["client_secret_post", "client_secret_basic"] | None = None
 
 
-class TokenExchangeConfig(BaseModel):
+class TokenExchangeConfig(HeaderCarrier):
     """OBO: swap the caller's live inbound token for a token bound to the upstream's audience. The
     gateway authenticates to the exchange endpoint as an OAuth client (`client_id`/`client_secret`);
     the inbound token is sent only to that endpoint, never to the upstream.
@@ -255,7 +304,7 @@ class ClientSecretAuth(BaseModel):
 ClientAuth = Annotated[PrivateKeyJwtAuth | ClientSecretAuth, Field(discriminator="source")]
 
 
-class IdJagConfig(BaseModel):
+class IdJagConfig(HeaderCarrier):
     """draft-ietf-oauth-identity-assertion-authz-grant (Okta "AI agent token exchange").
 
     Two legs: leg 1 is an RFC 8693 token exchange at the IdP org AS (`org_token_endpoint`) that
@@ -297,23 +346,16 @@ class Byok(BaseModel):
 ApiKeySource = Annotated[SharedKey | Byok, Field(discriminator="source")]
 
 
-class ApiKeyConfig(BaseModel):
+class ApiKeyConfig(HeaderCarrier):
     """A fixed credential injected as a header. The value is shared (in config) or seeded
-    per-user (pulled from the store); `header_name` and `value_prefix` say where and how it is
-    written, modeled like OpenAPI's apiKey scheme so any upstream convention is expressible
-    (Authorization + Bearer, a raw value on X-API-Key, Ocp-Apim-Subscription-Key, etc.).
+    per-user (pulled from the store); the inherited `header_name` and `value_prefix` say where
+    and how it is written.
     """
 
     model_config = ConfigDict(frozen=True)
     kind: Literal[AuthSpecKind.api_key] = AuthSpecKind.api_key
-    header_name: str = "Authorization"
-    value_prefix: str = "Bearer"
     key_source: ApiKeySource
 
-    def header(self, value: str) -> tuple[str, str]:
-        formatted: Final = f"{self.value_prefix} {value}" if self.value_prefix else value
-        return self.header_name, formatted
-
 
 class PassthroughConfig(BaseModel):
     """Client-driven upstream OAuth; the gateway forwards the client's upstream token."""

litellm/proxy/_experimental/mcp_server/server.py

@@ -433,7 +433,6 @@ if MCP_AVAILABLE:
         _client_forwarded_authorization_headers,
         _resolve_openapi_tool_auth,
         _should_strip_caller_authorization,
-        _without_authorization,
         global_mcp_server_manager,
     )
     from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
@@ -452,6 +451,7 @@ if MCP_AVAILABLE:
         split_server_prefix_from_name,
         strip_known_server_prefix,
     )
+    from litellm.types.mcp import DEFAULT_CREDENTIAL_HEADER, without_header
 
     ######################################################
     ############ MCP Tools List REST API Response Object #
@@ -1733,7 +1733,7 @@ if MCP_AVAILABLE:
                     raw_headers=raw_headers,
                     user_api_key_auth=user_api_key_auth,
                 ):
-                    extra_headers = _without_authorization(extra_headers)
+                    extra_headers = without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER)
         elif is_client_forwarded_mode:
             if not withhold_forwarded_authorization:
                 extra_headers = _client_forwarded_authorization_headers(

litellm/proxy/_lazy_openapi_snapshot.json

@@ -2689,6 +2689,11 @@
               "title": "Total Flat Cost",
               "type": "number"
             },
+            "total_gateway_injected_caching_savings_spend": {
+              "default": 0.0,
+              "title": "Total Gateway Injected Caching Savings Spend",
+              "type": "number"
+            },
             "total_pages": {
               "default": 1,
               "title": "Total Pages",
@@ -3175,6 +3180,11 @@
               "title": "Flat Cost",
               "type": "number"
             },
+            "gateway_injected_caching_savings_spend": {
+              "default": 0.0,
+              "title": "Gateway Injected Caching Savings Spend",
+              "type": "number"
+            },
             "prompt_caching_savings_spend": {
               "default": 0.0,
               "title": "Prompt Caching Savings Spend",
@@ -8744,7 +8754,7 @@
                 }
               ],
               "default": true,
-              "description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor' and 'generic_guardrail_api'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
+              "description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
               "title": "Fail On Error"
             },
             "guard_name": {
@@ -10238,6 +10248,18 @@
               "description": "AWS Bedrock runtime endpoint URL",
               "title": "Aws Bedrock Runtime Endpoint"
             },
+            "aws_external_id": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "External ID required by the target role's trust policy on sts:AssumeRole",
+              "title": "Aws External Id"
+            },
             "aws_profile_name": {
               "anyOf": [
                 {
@@ -10753,7 +10775,7 @@
                 }
               ],
               "default": true,
-              "description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor' and 'generic_guardrail_api'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
+              "description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
               "title": "Fail On Error"
             },
             "grounding_check": {
@@ -11352,6 +11374,18 @@
               "description": "Path to a JSON file containing ad-hoc recognizers for Presidio",
               "title": "Presidio Ad Hoc Recognizers"
             },
+            "presidio_analyze_chunk_size_bytes": {
+              "anyOf": [
+                {
+                  "type": "integer"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "Maximum UTF-8 bytes of text sent in a single Presidio /analyze call. Longer texts are split into overlapping chunks of at most this size and the merged results are remapped onto the original text. Defaults to 500000; set it below your analyzer deployment's request body limit, leaving headroom for the rest of the analyze payload.",
+              "title": "Presidio Analyze Chunk Size Bytes"
+            },
             "presidio_analyzer_api_base": {
               "anyOf": [
                 {
@@ -15038,6 +15072,17 @@
                 }
               ],
               "title": "Upstream Resource"
+            },
+            "upstream_token_header": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Upstream Token Header"
             }
           },
           "title": "MCPCredentials",
@@ -17518,6 +17563,17 @@
                 }
               ],
               "title": "Upstream Resource"
+            },
+            "upstream_token_header": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Upstream Token Header"
             }
           },
           "title": "MCPCredentials",
@@ -20352,6 +20408,17 @@
                 }
               ],
               "title": "Upstream Resource"
+            },
+            "upstream_token_header": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Upstream Token Header"
             }
           },
           "title": "MCPCredentials",
@@ -23699,6 +23766,17 @@
                 }
               ],
               "title": "Upstream Resource"
+            },
+            "upstream_token_header": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Upstream Token Header"
             }
           },
           "title": "MCPCredentials",
@@ -25193,6 +25271,9 @@
                       },
                       {
                         "$ref": "#/components/schemas/ChatCompletionImageObject"
+                      },
+                      {
+                        "$ref": "#/components/schemas/ChatCompletionToolReferenceObject"
                       }
                     ]
                   },
@@ -25280,6 +25361,26 @@
           "title": "ChatCompletionToolParamFunctionChunk",
           "type": "object"
         },
+        "ChatCompletionToolReferenceObject": {
+          "description": "Anthropic tool-search result block, carried through untouched so it survives a round trip.",
+          "properties": {
+            "tool_name": {
+              "title": "Tool Name",
+              "type": "string"
+            },
+            "type": {
+              "const": "tool_reference",
+              "title": "Type",
+              "type": "string"
+            }
+          },
+          "required": [
+            "type",
+            "tool_name"
+          ],
+          "title": "ChatCompletionToolReferenceObject",
+          "type": "object"
+        },
         "ChatCompletionUserMessage": {
           "properties": {
             "cache_control": {

litellm/proxy/_types.py

@@ -2589,6 +2589,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
         None,
         description="By default, the user calling /team/new is automatically added to the new team as a team admin. If True, proxy admins are no longer auto-added; members explicitly listed in members_with_roles are unaffected. Default is False.",
     )
+    enforce_fallback_model_access: bool | None = Field(
+        None,
+        description="If True, router fallbacks configured in router_settings are only attempted when the calling key (and its team and project) is allowed to call the fallback model; unauthorized fallback targets are skipped and the primary model's error is returned. Default is False.",
+    )
     scheduled_job_stagger: ScheduledJobStaggerSettings | None = Field(
         None,
         description=(
@@ -3577,6 +3581,7 @@ class SpendLogsMetadata(TypedDict):
     cost_breakdown: CostBreakdown | None  # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
     compression_savings: CompressionSavingsMetadata | None
     autorouter_savings: ReadOnly[float | None]  # stamped by the logging payload; None = not auto-routed
+    litellm_gateway_injected_cache: ReadOnly[str | None]
 
 
 class SpendLogsPayload(TypedDict):
@@ -4857,6 +4862,7 @@ class BaseDailySpendTransaction(TypedDict):
     # cost-savings metrics (dollars, priced per request before aggregation)
     compression_savings_spend: float
     prompt_caching_savings_spend: float
+    gateway_injected_caching_savings_spend: float  # writable-ok: the rollup queue accumulates into this key in place, as it does for every sibling spend field
     # Not required: rows queued by a pod running the previous release, or replayed from
     # the Redis buffer across an upgrade, carry no such key. Every reader coalesces a
     # missing value to zero, so requiring it here would describe a shape the aggregation

litellm/proxy/auth/fallback_model_access.py

@@ -0,0 +1,90 @@
+"""
+Authorize router fallback targets against the caller's key, team and project model access.
+
+`_enforce_key_and_fallback_model_access` only sees fallbacks the client sends in the request body.
+Fallbacks configured on the router (`router_settings.fallbacks` and friends) are chosen after auth,
+inside the router, so this predicate is injected into the router to re-run the same model access
+checks for each fallback target before it is attempted. Opt-in via
+`general_settings.enforce_fallback_model_access: true`.
+"""
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import Final
+
+from pydantic import BaseModel, ValidationError
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
+from litellm.router import Router
+
+
+class _RequestMetadata(BaseModel):
+    user_api_key_auth: UserAPIKeyAuth | None = None
+
+
+class _FallbackAccessSettings(BaseModel):
+    enforce_fallback_model_access: bool = False
+
+
+async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAuth, llm_router: Router) -> bool:
+    try:
+        await can_key_call_resolved_model(
+            model=model,
+            llm_model_list=None,
+            valid_token=valid_token,
+            llm_router=llm_router,
+        )
+    except ProxyException:
+        return False
+    except Exception as e:  # noqa: BLE001  # fail closed: a lookup failure must neither run the fallback nor replace the provider error
+        verbose_proxy_logger.warning("Skipping fallback to model=%s: authorization lookup failed: %s", model, e)
+        return False
+    return True
+
+
+def _token_in_metadata(metadata: object) -> UserAPIKeyAuth | None:
+    try:
+        return _RequestMetadata.model_validate(metadata).user_api_key_auth
+    except ValidationError:
+        return None
+
+
+def _user_api_key_auth_from_request(request_kwargs: Mapping[str, object]) -> UserAPIKeyAuth | None:
+    return next(
+        (
+            token
+            for field in ("metadata", "litellm_metadata")
+            if (token := _token_in_metadata(request_kwargs.get(field))) is not None
+        ),
+        None,
+    )
+
+
+def _enforced_by_general_settings() -> bool:
+    from litellm.proxy.proxy_server import general_settings
+
+    return _FallbackAccessSettings.model_validate(general_settings).enforce_fallback_model_access
+
+
+@dataclass(frozen=True, slots=True)
+class RouterFallbackAccessCheck:
+    """
+    `FallbackAccessCheck` for the proxy's router: while `is_enforced()` is true, a fallback target
+    is attempted only when the key behind the request could have requested it directly. Requests
+    that carry no key (for example internal health checks) are not restricted.
+    """
+
+    is_enforced: Callable[[], bool]
+
+    async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
+        if not self.is_enforced():
+            return True
+        valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
+        if valid_token is None:
+            return True
+        return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=llm_router)
+
+
+router_fallback_access_check: Final = RouterFallbackAccessCheck(is_enforced=_enforced_by_general_settings)

litellm/proxy/common_request_processing.py

@@ -417,15 +417,6 @@ def _litellm_model_supports_stream_options(litellm_model: str) -> bool:
     return supported_params is not None and "stream_options" in supported_params
 
 
-def _deployment_litellm_model(deployment: Mapping[str, object]) -> str | None:
-    litellm_params: Final = deployment.get("litellm_params")
-    if isinstance(litellm_params, Mapping):
-        litellm_model = litellm_params.get("model")
-    else:
-        litellm_model = getattr(litellm_params, "model", None)
-    return litellm_model if isinstance(litellm_model, str) else None
-
-
 def _model_deployments_support_stream_options(
     model: object,
     llm_router: Router | None,
@@ -433,11 +424,8 @@ def _model_deployments_support_stream_options(
 ) -> bool:
     if not isinstance(model, str):
         return False
-    deployments = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router is not None else None
-    deployment_models: Final = tuple(
-        litellm_model
-        for deployment in deployments or ()
-        if (litellm_model := _deployment_litellm_model(deployment)) is not None
+    deployment_models: Final = (
+        llm_router.resolved_litellm_models(model, team_id=team_id) if llm_router is not None else ()
     )
     candidate_models: Final = deployment_models if deployment_models else (model,)
     return all(_litellm_model_supports_stream_options(m) for m in candidate_models)

litellm/proxy/config_resolvers/alerting.py

@@ -25,3 +25,7 @@ EMAIL_DESCRIPTORS: Final[tuple[FieldDescriptor, ...]] = (
 SLACK_DESCRIPTORS: Final[tuple[FieldDescriptor, ...]] = (
     FieldDescriptor("SLACK_WEBHOOK_URL", "SLACK_WEBHOOK_URL", "SLACK_WEBHOOK_URL", is_secret=True),
 )
+
+MS_TEAMS_DESCRIPTORS: Final[tuple[FieldDescriptor, ...]] = (
+    FieldDescriptor("MS_TEAMS_WEBHOOK_URL", "MS_TEAMS_WEBHOOK_URL", "MS_TEAMS_WEBHOOK_URL", is_secret=True),
+)

litellm/proxy/db/daily_spend_bulk_upsert.py

@@ -62,6 +62,7 @@ _SPEND_COLUMNS: Final = (
     "spend",
     "compression_savings_spend",
     "prompt_caching_savings_spend",
+    "gateway_injected_caching_savings_spend",
     "autorouter_savings_spend",
 )
 

litellm/proxy/db/db_spend_update_writer.py

@@ -62,6 +62,7 @@ from litellm.proxy.spend_tracking.savings import (
     compute_savings_spend,
     extract_cache_creation_tokens,
     extract_cache_read_tokens,
+    marks_gateway_injection,
 )
 from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
 from litellm.repositories.prisma_protocols import BatchTable
@@ -315,6 +316,7 @@ class DBSpendUpdateWriter:
                 model=payload.get("model"),
                 custom_llm_provider=payload.get("custom_llm_provider"),
                 compression_saved_tokens=0,
+                gateway_injected_cache=marks_gateway_injection(metadata, payload.get("model_id")),
                 routing_decision=metadata.get("routing_decision"),
                 usage_object=usage_object_raw if isinstance(usage_object_raw, dict) else None,
                 model_id=payload.get("model_id"),
@@ -1879,6 +1881,7 @@ class DBSpendUpdateWriter:
                 model=payload.get("model", None),
                 custom_llm_provider=payload.get("custom_llm_provider", None),
                 compression_saved_tokens=compression_saved_tokens,
+                gateway_injected_cache=marks_gateway_injection(_metadata, payload.get("model_id")),
                 routing_decision=_metadata.get("routing_decision"),
                 model_id=payload.get("model_id"),
                 llm_router=_get_llm_router,
@@ -1911,6 +1914,7 @@ class DBSpendUpdateWriter:
                 compression_saved_tokens=compression_saved_tokens,
                 compression_savings_spend=savings_spend.compression,
                 prompt_caching_savings_spend=savings_spend.prompt_caching,
+                gateway_injected_caching_savings_spend=savings_spend.gateway_injected_caching,
                 autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter,
             )
             return daily_transaction

litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py

@@ -134,6 +134,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
                         payload.get("prompt_caching_savings_spend", 0) or 0
                     ) + daily_transaction.get("prompt_caching_savings_spend", 0)
 
+                    daily_transaction["gateway_injected_caching_savings_spend"] = (
+                        payload.get("gateway_injected_caching_savings_spend", 0) or 0
+                    ) + daily_transaction.get("gateway_injected_caching_savings_spend", 0)
+
                     daily_transaction["autorouter_savings_spend"] = (
                         payload.get("autorouter_savings_spend", 0) or 0
                     ) + daily_transaction.get("autorouter_savings_spend", 0)

litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

@@ -686,6 +686,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         aws_profile_name: Final = self.optional_params.get("aws_profile_name", None)
         aws_web_identity_token: Final = self.optional_params.get("aws_web_identity_token", None)
         aws_sts_endpoint: Final = self.optional_params.get("aws_sts_endpoint", None)
+        aws_external_id: Final = self.optional_params.get("aws_external_id", None)
 
         ### SET REGION NAME ###
         aws_region_name = self.get_aws_region_name_for_non_llm_api_calls(
@@ -702,6 +703,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
             aws_role_name=aws_role_name,
             aws_web_identity_token=aws_web_identity_token,
             aws_sts_endpoint=aws_sts_endpoint,
+            aws_external_id=aws_external_id,
         )
         return credentials, aws_region_name
 

litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/__init__.py

@@ -25,6 +25,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
             GuardrailEventHooks.post_call.value,
         ],
         default_on=litellm_params.default_on,
+        fail_on_error=litellm_params.fail_on_error,
     )
     litellm.logging_callback_manager.add_litellm_callback(_crowdstrike_aidr_callback)
 

litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py

@@ -1,10 +1,11 @@
 import json
 import os
+import time
 from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple, Optional, cast
 
 from fastapi import HTTPException
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
 from typing_extensions import Any, override
 
 from litellm._logging import verbose_proxy_logger
@@ -142,7 +143,7 @@ def _extract_text_from_message(message: _Message) -> str:
     return "\n".join(part.text for part in content if isinstance(part, _TextContentPart))
 
 
-def _merge_metadata_bags(request_data: Mapping[str, Any]) -> dict[str, Any] | None:
+def _merge_metadata_bags(request_data: Mapping[str, Any]) -> Mapping[str, Any] | None:
     merged: Final[dict[str, Any]] = {}
     present = False
     for bag in (request_data.get("metadata"), request_data.get("litellm_metadata")):
@@ -153,7 +154,7 @@ def _merge_metadata_bags(request_data: Mapping[str, Any]) -> dict[str, Any] | No
 
 
 def _messages_since_last_assistant(
-    messages: list[AllMessageValues],
+    messages: Sequence[AllMessageValues],
 ) -> _FilteredMessages:
     if not messages:
         return _FilteredMessages([], ())
@@ -239,6 +240,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail):
         guardrail_name: str,
         api_key: str | None = None,
         api_base: str | None = None,
+        fail_on_error: bool | None = True,
         **kwargs,
     ) -> None:
         """
@@ -251,6 +253,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail):
             **kwargs: Additional arguments passed to the CustomGuardrail base class.
         """
         self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
+        self.fail_on_error = True if fail_on_error is None else fail_on_error
 
         self.api_key = api_key or os.environ.get("CS_AIDR_TOKEN")
         if not self.api_key:
@@ -306,11 +309,13 @@ class CrowdStrikeAIDRHandler(CustomGuardrail):
         assert response is not None
         response.raise_for_status()
 
-        result = _GuardChatCompletionsResponse.model_validate(response.json()).result or _GuardChatCompletionsResult()
+        response_body: Final[object] = response.json()
+        raw_result: Final[object] = response_body.get("result") if isinstance(response_body, dict) else None
+        blocked_signal: Final[object] = raw_result.get("blocked") if isinstance(raw_result, dict) else None
 
-        if result.blocked:
+        if blocked_signal:
             verbose_proxy_logger.warning(
-                "CrowdStrike AIDR Guardrail (%s): Request blocked. Response: %s", hook_name, result
+                "CrowdStrike AIDR Guardrail (%s): Request blocked. Verdict: %s", hook_name, blocked_signal
             )
             raise HTTPException(
                 status_code=400,  # Bad Request, indicating violation
@@ -319,6 +324,23 @@ class CrowdStrikeAIDRHandler(CustomGuardrail):
                     "guardrail_name": self.guardrail_name,
                 },
             )
+
+        try:
+            result: Final = (
+                _GuardChatCompletionsResponse.model_validate(response_body).result or _GuardChatCompletionsResult()
+            )
+        except ValidationError as validation_error:
+            transformed_signal: Final[object] = raw_result.get("transformed") if isinstance(raw_result, dict) else None
+            if transformed_signal:
+                raise HTTPException(
+                    status_code=500,
+                    detail={  # mutable-ok: one-shot HTTPException detail payload, never mutated after construction
+                        "error": "CrowdStrike AIDR returned a transformed response litellm could not parse; "
+                        "failing closed instead of dropping the delivered redactions",
+                        "guardrail_name": self.guardrail_name,
+                    },
+                ) from validation_error
+            raise
         verbose_proxy_logger.debug(
             "CrowdStrike AIDR Guardrail (%s): Request passed. Response: %s", hook_name, result.detectors
         )
@@ -362,6 +384,34 @@ class CrowdStrikeAIDRHandler(CustomGuardrail):
         tail: Final = guard_output.messages[-num_assistant_messages:] if num_assistant_messages > 0 else []
         return [_extract_text_from_message(msg) for msg in tail]
 
+    async def _call_or_fail_open(
+        self, payload: dict[str, Any], hook_name: str, request_data: dict
+    ) -> _GuardChatCompletionsResult:
+        start_time: Final = time.time()
+        try:
+            return await self._call_crowdstrike_aidr_guard(payload, hook_name)
+        except HTTPException:
+            raise
+        except Exception as error:
+            if self.fail_on_error:
+                raise
+            verbose_proxy_logger.error(
+                "CrowdStrike AIDR Guardrail failed open | hook_name: %s error: %s",
+                hook_name,
+                error,
+                exc_info=True,
+            )
+            end_time: Final = time.time()
+            self.add_standard_logging_guardrail_information_to_request_data(
+                guardrail_json_response=error,
+                request_data=request_data,
+                guardrail_status="guardrail_failed_to_respond",
+                start_time=start_time,
+                end_time=end_time,
+                duration=end_time - start_time,
+            )
+            return _GuardChatCompletionsResult()
+
     @override
     def structured_messages_cover_full_request(self) -> bool:
         return effective_skip_system_message_for_guardrail(self) or effective_skip_tool_message_for_guardrail(self)
@@ -439,7 +489,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail):
                 extra_info["user_name"] = user_email
             ai_guard_payload["extra_info"] = extra_info
 
-        result: Final = await self._call_crowdstrike_aidr_guard(ai_guard_payload, hook_name)
+        result: Final = await self._call_or_fail_open(ai_guard_payload, hook_name, request_data)
 
         if "body" in request_data or "messages" in request_data:
             add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name)

litellm/proxy/guardrails/guardrail_hooks/presidio.py

@@ -11,7 +11,7 @@
 import asyncio
 import json
 import threading
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Sequence
 from contextlib import asynccontextmanager
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, cast
@@ -22,6 +22,11 @@ from typing_extensions import NotRequired, ReadOnly
 import litellm
 from litellm import get_secret
 from litellm._logging import verbose_proxy_logger
+from litellm.constants import (
+    DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES,
+    PRESIDIO_ANALYZE_CHUNK_CONCURRENCY,
+    PRESIDIO_ANALYZE_CHUNK_OVERLAP_CHARS,
+)
 from litellm.types.utils import GenericGuardrailAPIInputs
 
 if TYPE_CHECKING:
@@ -63,6 +68,18 @@ class _PresidioAnonymizeResponse(TypedDict):
     items: ReadOnly[NotRequired[list[_PresidioAnonymizeItem]]]
 
 
+_LoopSemaphores = dict[asyncio.AbstractEventLoop, asyncio.Semaphore]
+
+
+def _json_escaped_len(text: str) -> int:
+    """
+    Byte length of ``text`` as it appears serialized inside the JSON request
+    body sent to Presidio (``json.dumps`` escapes non-ASCII characters, so a
+    3-byte UTF-8 character can occupy 6+ bytes on the wire).
+    """
+    return len(json.dumps(text).encode("utf-8")) - 2  # strip the surrounding quotes
+
+
 class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
     user_api_key_cache = None
     ad_hoc_recognizers: list[str] | None = None
@@ -93,6 +110,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         presidio_language: str | None = None,
         presidio_score_thresholds: dict[PiiEntityType | str, float] | None = None,
         presidio_entities_deny_list: list[PiiEntityType | str] | None = None,
+        presidio_analyze_chunk_size_bytes: int | None = None,
         **kwargs,
     ):
         if logging_only is True:
@@ -121,6 +139,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         self.presidio_score_thresholds: dict[PiiEntityType | str, float] = presidio_score_thresholds or {}
         self.presidio_entities_deny_list: list[PiiEntityType | str] = presidio_entities_deny_list or []
         self.presidio_language = presidio_language or "en"
+        self.presidio_analyze_chunk_size_bytes: int = self._coerce_analyze_chunk_size(presidio_analyze_chunk_size_bytes)
         # Shared HTTP session to prevent memory leaks (issue #14540)
         self._http_session: aiohttp.ClientSession | None = None
         # Lock to prevent race conditions when creating session under concurrent load
@@ -134,6 +153,10 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         # Loop-bound session cache for background threads
         self._loop_sessions: dict[asyncio.AbstractEventLoop, aiohttp.ClientSession] = {}
 
+        # Per-loop semaphores bounding chunked-analyze fan-out across ALL
+        # concurrent oversized blocks/requests on this instance, not per call
+        self._loop_chunk_semaphores: _LoopSemaphores = {}  # mutable-ok: per-loop semaphore cache
+
         if mock_testing is True:  # for testing purposes only
             return
 
@@ -280,7 +303,28 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
     ) -> list[PresidioAnalyzeResponseItem] | _PresidioAnonymizeResponse:
         """
         Send text to the Presidio analyzer endpoint and get analysis results
+
+        Texts larger than ``presidio_analyze_chunk_size_bytes`` (UTF-8) are split
+        into overlapping chunks, analyzed per chunk, and the per-chunk results
+        are remapped onto the original text. Presidio analyzer deployments
+        commonly cap the /analyze request body size (e.g. at 1 MB), and analyzer
+        latency grows with payload size.
         """
+        # Chunk oversized texts before the try block so that a failing chunk
+        # keeps the same sanitized error message a single call would produce.
+        # A single-character text can never be split further, so it always
+        # takes the single-call path regardless of its encoded width.
+        if (
+            text
+            and len(text) > 1
+            and self.mock_redacted_text is None
+            and _json_escaped_len(text) > self.presidio_analyze_chunk_size_bytes
+        ):
+            return await self._analyze_text_chunked(
+                text=text,
+                presidio_config=presidio_config,
+                request_data=request_data,
+            )
         try:
             # Skip empty or whitespace-only text to avoid Presidio errors
             # Common in tool/function calling where assistant content is empty
@@ -397,6 +441,201 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
             # contain API keys or other secrets) in error responses.
             raise Exception(f"Presidio PII analysis failed: {type(e).__name__}") from e
 
+    async def _analyze_text_chunked(
+        self,
+        text: str,
+        presidio_config: PresidioPerRequestConfig | None,
+        request_data: dict,  # mutable-ok: shared per-request state dict, matching analyze_text's parameter
+    ) -> list[PresidioAnalyzeResponseItem]:  # mutable-ok: analyze_text's declared return type requires list
+        """
+        Analyze an oversized text by splitting it into overlapping chunks.
+
+        Each chunk serializes to at most ``presidio_analyze_chunk_size_bytes``
+        bytes inside the JSON request body, so every /analyze call stays below
+        the analyzer deployment's request body limit; per-chunk results are remapped onto the original text and
+        merged. Raises exactly like a single ``analyze_text`` call if any chunk
+        fails.
+
+        Only the analyzer side is chunked: the later anonymize call still
+        receives the full original text, so texts above the anonymizer's own
+        body limit that contain detections keep failing there.
+        """
+        text_chunks: Final = self._split_text_for_analysis(
+            text=text,
+            chunk_size_bytes=self.presidio_analyze_chunk_size_bytes,
+            overlap_chars=PRESIDIO_ANALYZE_CHUNK_OVERLAP_CHARS,
+        )
+        verbose_proxy_logger.debug(
+            "Presidio analyze: text exceeds %s bytes, analyzing in %s overlapping chunks",
+            self.presidio_analyze_chunk_size_bytes,
+            len(text_chunks),
+        )
+        # Bound the fan-out so oversized requests cannot saturate the analyzer.
+        # The semaphore is shared per event loop across every chunked call on
+        # this instance, so many oversized blocks in one request (or many
+        # concurrent requests) still hold at most this many analyzer calls in
+        # flight. On the proxy's main thread the shared-session lock in
+        # _get_session_iterator additionally serializes the HTTP calls; the
+        # bound matters for loop-bound sessions (background threads).
+        analyze_semaphore: Final = self._get_chunk_semaphore()
+
+        async def _analyze_chunk_bounded(
+            chunk_text: str,
+        ) -> Sequence[PresidioAnalyzeResponseItem] | _PresidioAnonymizeResponse:
+            async with analyze_semaphore:
+                return await self.analyze_text(
+                    text=chunk_text,
+                    presidio_config=presidio_config,
+                    request_data=request_data,
+                )
+
+        gathered: Final = await asyncio.gather(
+            *(_analyze_chunk_bounded(chunk_text) for _, chunk_text in text_chunks),
+            return_exceptions=True,
+        )
+        chunk_results: Final = []
+        for result in gathered:
+            if isinstance(result, BaseException):
+                raise result
+            # analyze_text only returns a non-list shape when mock_redacted_text
+            # is set, and the chunked path is never entered in that case.
+            typed_result = cast("list[PresidioAnalyzeResponseItem]", result)  # cast-ok: gather() erases element type
+            # Apply the configured score thresholds and deny list BEFORE the
+            # overlap merge: a below-threshold detection must not win overlap
+            # resolution against one the thresholds would keep. The same filter
+            # runs again downstream in check_pii, where it is a no-op for the
+            # already-filtered items.
+            filtered_result = self.filter_analyze_results_by_score(analyze_results=typed_result)
+            chunk_results.append(
+                cast("list[PresidioAnalyzeResponseItem]", filtered_result)  # cast-ok: list input yields list
+            )
+        return self._merge_chunked_analyze_results(text_chunks=text_chunks, chunk_results=chunk_results)
+
+    def _get_chunk_semaphore(self) -> asyncio.Semaphore:
+        """Per-event-loop semaphore shared by all chunked analyze calls on this instance."""
+        loop: Final = asyncio.get_running_loop()
+        existing: Final = self._loop_chunk_semaphores.get(loop)
+        if existing is not None:
+            return existing
+        created: Final = asyncio.Semaphore(PRESIDIO_ANALYZE_CHUNK_CONCURRENCY)
+        self._loop_chunk_semaphores[loop] = created
+        return created
+
+    @staticmethod
+    def _coerce_analyze_chunk_size(value: int | None) -> int:
+        """
+        Validate a configured chunk size, falling back to the default.
+
+        Non-positive values would either bypass chunking entirely or degenerate
+        it into per-character splits (silently disabling detection), so they are
+        replaced by the default; values below 4 bytes are floored to 4 and the
+        splitter always emits at least one character per chunk, so the chunked
+        path can never re-enter itself.
+        """
+        if not value or value <= 0:
+            return DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES
+        return max(value, 4)
+
+    @staticmethod
+    def _split_text_for_analysis(
+        text: str,
+        chunk_size_bytes: int,
+        overlap_chars: int,
+    ) -> Sequence[tuple[int, str]]:
+        """
+        Split ``text`` into chunks whose JSON-serialized form is at most
+        ``chunk_size_bytes`` bytes (the analyzer body limit applies to the
+        JSON request body, where non-ASCII characters are escaped and larger
+        than their raw UTF-8 encoding).
+
+        Consecutive chunks overlap by up to ``overlap_chars`` characters so a
+        PII entity up to that length lying across a chunk boundary is still
+        seen whole by one of the chunks (longer boundary-straddling entities
+        may be seen only truncated); ``_merge_chunked_analyze_results`` resolves
+        the duplicate and truncated detections this produces. Returns
+        ``(char_offset, chunk_text)`` pairs where ``char_offset`` is the
+        chunk's start position in the original text.
+        """
+        chunks: Final = []
+        text_len: Final = len(text)
+        start = 0  # rebind-ok: chunk cursor advances across the loop
+        while start < text_len:
+            # Serialized length of a character is at least 1 byte, so a slice
+            # of chunk_size_bytes characters is a sufficient search window.
+            candidate = text[start : start + chunk_size_bytes]
+            if _json_escaped_len(candidate) <= chunk_size_bytes:
+                chunk = candidate
+            else:
+                # Largest prefix whose serialized form fits the budget.
+                low, high = 1, len(candidate)
+                while low < high:
+                    mid = (low + high + 1) // 2
+                    if _json_escaped_len(candidate[:mid]) <= chunk_size_bytes:
+                        low = mid
+                    else:
+                        high = mid - 1
+                # low >= 1 keeps the loop advancing even when a single
+                # character serializes over a (floored, tiny) budget.
+                chunk = candidate[:low]
+            end = start + len(chunk)
+            chunks.append((start, chunk))
+            if end >= text_len:
+                break
+            # Cap the overlap so the next chunk always makes forward progress.
+            effective_overlap = min(overlap_chars, len(chunk) // 2)
+            start = max(start + 1, end - effective_overlap)
+        return chunks
+
+    @staticmethod
+    def _merge_chunked_analyze_results(
+        text_chunks: Sequence[tuple[int, str]],
+        chunk_results: Sequence[Sequence[PresidioAnalyzeResponseItem]],
+    ) -> list[PresidioAnalyzeResponseItem]:  # mutable-ok: analyze_text's declared return type requires list
+        """
+        Remap per-chunk analyzer offsets onto the original text and merge.
+
+        A detection in an overlap region is reported by both neighbouring
+        chunks, and a boundary entity can additionally be reported truncated by
+        the chunk that saw only its head or tail. Same-entity-type detections
+        with overlapping remapped spans are therefore resolved by keeping the
+        longest span (highest score on ties) — mirroring the same-type conflict
+        removal Presidio's AnalyzerEngine applies within a single call, and
+        keeping overlapping spans from corrupting the numbered-token rewriter.
+        Detections of DIFFERENT entity types may still overlap, exactly as in a
+        single-call response. The merged list is sorted by position.
+        """
+        remapped: Final = []
+        for (char_offset, _), results in zip(text_chunks, chunk_results, strict=True):
+            for item in results:
+                item_start = item.get("start")
+                item_end = item.get("end")
+                if item_start is not None:
+                    item["start"] = item_start + char_offset
+                if item_end is not None:
+                    item["end"] = item_end + char_offset
+                remapped.append(item)
+
+        def _priority(item: PresidioAnalyzeResponseItem) -> tuple[int, float]:
+            span_start: Final = item.get("start") or 0
+            span_end: Final = item.get("end") or 0
+            return (-(span_end - span_start), -(item.get("score") or 0.0))
+
+        merged: Final = []
+        kept_spans_by_type: Final = {}
+        for item in sorted(remapped, key=_priority):
+            item_start = item.get("start")
+            item_end = item.get("end")
+            if item_start is None or item_end is None:
+                merged.append(item)
+                continue
+            kept_spans = kept_spans_by_type.setdefault(str(item.get("entity_type")), [])
+            if any(item_start < kept_end and kept_start < item_end for kept_start, kept_end in kept_spans):
+                continue
+            kept_spans.append((item_start, item_end))
+            merged.append(item)
+        merged.sort(key=lambda r: (r.get("start") or 0, r.get("end") or 0))
+        return merged
+
     async def _post_presidio_anonymize(
         self,
         text: str,
@@ -1392,3 +1631,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
             self.presidio_score_thresholds = litellm_params.presidio_score_thresholds
         if litellm_params.presidio_entities_deny_list:
             self.presidio_entities_deny_list = litellm_params.presidio_entities_deny_list
+        if litellm_params.presidio_analyze_chunk_size_bytes is not None:
+            # Same validation as __init__: a non-positive value from a guardrail
+            # update must not silently disable detection via degenerate chunking.
+            self.presidio_analyze_chunk_size_bytes = self._coerce_analyze_chunk_size(
+                litellm_params.presidio_analyze_chunk_size_bytes
+            )

litellm/proxy/guardrails/guardrail_initializers.py

@@ -34,6 +34,7 @@ def initialize_bedrock(litellm_params: LitellmParams, guardrail: Guardrail):
         aws_role_name=litellm_params.aws_role_name,
         aws_web_identity_token=litellm_params.aws_web_identity_token,
         aws_sts_endpoint=litellm_params.aws_sts_endpoint,
+        aws_external_id=litellm_params.aws_external_id,
         aws_bedrock_runtime_endpoint=litellm_params.aws_bedrock_runtime_endpoint,
         experimental_use_latest_role_message_only=litellm_params.experimental_use_latest_role_message_only,
         only_scan_new_messages=litellm_params.only_scan_new_messages or False,
@@ -103,7 +104,12 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail):
             apply_to_output=False,
         )
         params.update(overrides)
-        callback: Final = _OPTIONAL_PresidioPIIMasking(**params)
+        # Passed outside the heterogeneous params dict so the argument keeps
+        # its precise int | None type.
+        callback: Final = _OPTIONAL_PresidioPIIMasking(
+            presidio_analyze_chunk_size_bytes=litellm_params.presidio_analyze_chunk_size_bytes,
+            **params,
+        )
         litellm.logging_callback_manager.add_litellm_callback(callback)
         return callback
 

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -1,5 +1,6 @@
 import asyncio
 import copy
+import json
 import logging
 import os
 import secrets
@@ -11,10 +12,16 @@ from typing import Any, Final, Literal, TypedDict, cast
 
 import fastapi
 from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
+from typing_extensions import ReadOnly
 
 import litellm
 from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm.constants import HEALTH_CHECK_TIMEOUT_SECONDS
+from litellm.integrations.SlackAlerting.ms_teams import (
+    MS_TEAMS_ALERT_HEADERS,
+    build_ms_teams_payload,
+    get_ms_teams_webhook_url,
+)
 from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.proxy._types import (
@@ -164,6 +171,7 @@ services = (
         "langfuse",
         "langfuse_otel",
         "slack",
+        "ms_teams",
         "openmeter",
         "webhook",
         "email",
@@ -180,6 +188,15 @@ services = (
 )
 
 
+class _ServiceTestErrorDetail(TypedDict):
+    error: ReadOnly[str]
+
+
+class _ServiceTestSuccessResponse(TypedDict):
+    status: ReadOnly[str]
+    message: ReadOnly[str]
+
+
 @router.get(
     "/test",
     tags=["health"],
@@ -238,6 +255,7 @@ async def health_services_endpoint(
             "langfuse",
             "langfuse_otel",
             "slack",
+            "ms_teams",
             "openmeter",
             "webhook",
             "braintrust",
@@ -448,6 +466,38 @@ async def health_services_endpoint(
                     status_code=422,
                     detail={"error": f'"{service}" not in proxy config: general_settings. Unable to test this.'},
                 )
+        if service == "ms_teams":
+            if "ms_teams" not in general_settings.get("alerting", ()):
+                not_configured_detail: Final[_ServiceTestErrorDetail] = {
+                    "error": f'"{service}" not in proxy config: general_settings. Unable to test this.'
+                }
+                raise HTTPException(status_code=422, detail=not_configured_detail)
+            ms_teams_webhook_url: Final = get_ms_teams_webhook_url()
+            if ms_teams_webhook_url is None:
+                missing_webhook_detail: Final[_ServiceTestErrorDetail] = {
+                    "error": "MS_TEAMS_WEBHOOK_URL not set. Unable to test this."
+                }
+                raise HTTPException(status_code=422, detail=missing_webhook_detail)
+            ms_teams_test_message: Final = (
+                f"Alert type: `{AlertType.budget_alerts.value}`\nLevel: `Low`\n"
+                f"Timestamp: `{datetime.now().strftime('%H:%M:%S')}`\n\n"
+                "Message: This is a test MS Teams alert message"
+            )
+            ms_teams_response: Final = await proxy_logging_obj.slack_alerting_instance.async_http_handler.post(
+                url=ms_teams_webhook_url,
+                headers=dict(MS_TEAMS_ALERT_HEADERS),  # mutable-ok: async_http_handler.post only accepts dict headers
+                data=json.dumps(build_ms_teams_payload(ms_teams_test_message)),
+            )
+            if ms_teams_response.status_code >= 400:
+                delivery_failed_detail: Final[_ServiceTestErrorDetail] = {
+                    "error": f"MS Teams webhook returned status {ms_teams_response.status_code}: {ms_teams_response.text}"
+                }
+                raise HTTPException(status_code=500, detail=delivery_failed_detail)
+            ms_teams_success: Final[_ServiceTestSuccessResponse] = {
+                "status": "success",
+                "message": "Mock MS Teams Alert sent, verify MS Teams Alert Received in your channel",
+            }
+            return ms_teams_success
         if service == "email":
             webhook_event: Final = WebhookEvent(
                 event="key_created",

litellm/proxy/litellm_pre_call_utils.py

@@ -51,6 +51,7 @@ from litellm.proxy.common_utils.callback_utils import (
     strip_callback_config,
 )
 from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers
+from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY
 
 # Cache special headers as a frozenset for O(1) lookup performance
 _SPECIAL_HEADERS_CACHE: Final = frozenset(v.value.lower() for v in SpecialHeaders._member_map_.values())
@@ -221,6 +222,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = (
     "policy_sources",
     "guardrail_scan_ids",
     "routing_decision",
+    GATEWAY_INJECTED_CACHE_METADATA_KEY,
     "pillar_response_headers",
     "_guardrail_pipelines",
     "_pipeline_managed_guardrails",
@@ -275,6 +277,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
     "policy_sources",
     "guardrail_scan_ids",
     "routing_decision",
+    GATEWAY_INJECTED_CACHE_METADATA_KEY,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -1,7 +1,7 @@
 """
 AUTO ROUTER MANAGEMENT ENDPOINTS
 
-POST /auto_router/test_routing - Route one prompt through an unsaved complexity-router config
+POST /auto_router/test_routing - Route one request through an unsaved complexity-router config
 POST /auto_router/validate_complexity_router_config - Dry-run the complexity-router write gate without saving
 """
 
@@ -17,7 +17,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator
 
 from litellm._logging import verbose_proxy_logger
 from litellm.exceptions import BudgetExceededError
-from litellm.litellm_core_utils.llm_judge import router_resolves_model
+from litellm.litellm_core_utils.llm_judge import judge_target
 from litellm.proxy._types import (
     CommonProxyErrors,
     LiteLLM_TeamTable,
@@ -32,11 +32,18 @@ from litellm.proxy.auth.auth_checks import (
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.db.autorouter_session_rollup import AUTOROUTER_BENCHMARKS_SQL
-from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
+from litellm.proxy.litellm_pre_call_utils import (
+    LiteLLMProxyRequestSetup,
+    refresh_proxy_server_request_body_snapshot,
+)
 from litellm.repositories.base_repository import SupportsModelDump
 from litellm.repositories.team_repository import TeamRepository
 from litellm.router_strategy.complexity_router import ComplexityRouter
-from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model
+from litellm.router_utils.auto_router_model_naming import (
+    StrategyRouterDependencyRole,
+    classify_strategy_router_model,
+    strategy_router_dependencies,
+)
 from litellm.types.management_endpoints.auto_router_endpoints import (
     SHADOW_EVAL_TURN_VALVE,
     AutoRouterBenchmarkGroup,
@@ -86,6 +93,9 @@ class _VerificationTokenRow(Protocol):
     @property
     def key_name(self) -> str | None: ...
 
+    @property
+    def team_id(self) -> str | None: ...
+
 
 class _VerificationTokenTable(Protocol):
     async def find_unique(self, *, where: Mapping[str, object]) -> _VerificationTokenRow | None: ...
@@ -285,19 +295,30 @@ async def preview_auto_router_routing(
     user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
 ) -> AutoRouterRoutingTestResponse:
     """
-    Route a single prompt through a complexity-router config and report where it landed.
+    Route a single request through a complexity-router config and report where it landed.
 
-    Answers "which model would this prompt get?" for a config that only exists in a form,
-    so an auto router can be checked before it is created. The prompt is classified by the
-    same pre-routing hook a live request runs, then dropped: nothing is sent to the model it
-    routed to, and no auto router is created. A heuristic config therefore spends nothing, while
-    an `llm` classifier or semantic keyword matching bills its classifier/embedding call to the
-    calling key, like Test Connection does.
+    Answers "which model would this request get?" for a config that only exists in a form,
+    so an auto router can be checked before it is created. The request is classified by the
+    same pre-routing hook a live request runs, over the same messages, system prompt and tool
+    definitions, then dropped: nothing is sent to the model it routed to, and no auto router is
+    created. A heuristic config therefore spends nothing, while an `llm` classifier or semantic
+    keyword matching bills its classifier/embedding call to the calling key, like Test Connection
+    does.
+
+    Send `messages` to classify a real turn, with `system` and `tools` beside it when the surface
+    carries them top level, as Anthropic /v1/messages does. `prompt` is the single-ask shorthand and
+    routes as one user turn with nothing around it.
 
     **Example Request:**
     ```json
     {
-        "prompt": "think step by step about how to shard this table",
+        "messages": [
+            {"role": "system", "content": "You are a database migration assistant"},
+            {"role": "user", "content": "the index is not unique"},
+            {"role": "assistant", "content": "Then two workers can both insert. Add a unique index"},
+            {"role": "user", "content": "ok do it"}
+        ],
+        "tools": [{"type": "function", "function": {"name": "Bash", "description": "Run a command"}}],
         "complexity_router_config": {
             "tiers": {"SIMPLE": ["gpt-4o-mini"], "REASONING": ["o3"]},
             "classifier_type": "heuristic"
@@ -340,18 +361,21 @@ async def preview_auto_router_routing(
     )
 
     request_kwargs: Final = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
-        data={"metadata": {}},  # mutable-ok: the request-metadata helper takes and returns request kwargs as a dict
+        data={  # mutable-ok: the request-metadata helper takes and returns request kwargs as a dict
+            **data.wire_body(),
+            "metadata": {},  # mutable-ok: the request-metadata helper writes the auth fields into this dict
+            "proxy_server_request": {"body": None},  # mutable-ok: the snapshot owner fills body in place
+        },
         user_api_key_dict=user_api_key_dict,
         _metadata_variable_name="metadata",
     )
+    refresh_proxy_server_request_body_snapshot(request_kwargs)
 
     try:
         hook_response: Final = await complexity_router.async_pre_routing_hook(
             model=data.router_name,
             request_kwargs=request_kwargs,
-            messages=[  # mutable-ok: the routing hook's signature takes a list of message dicts
-                {"role": "user", "content": data.prompt},  # mutable-ok: a message is dict-shaped
-            ],
+            messages=request_kwargs["messages"],
         )
     except Exception as e:  # noqa: BLE001 -- surfaces any classifier/plugin failure to the caller as a 400 instead of a 500, since the config under test is caller input
         verbose_proxy_logger.exception("Auto router routing test failed. Due to error - %s", e)
@@ -654,30 +678,126 @@ def _is_configured_pre_routing_strategy(llm_router: "Router", router_name: str)
     )
 
 
-def _validate_plain_model(llm_router: "Router | None", model: str, field_name: str) -> None:
+def _validate_plain_model(
+    llm_router: "Router | None", model: str, field_name: str, team_ids: Sequence[str | None]
+) -> None:
     """Reject a model the dispatch path cannot resolve, at start rather than as a silently
     growing error count once the job is already sampling and billing. Both the judge and a
     reverse job's baseline must be plain models: an auto-router in either slot would
-    re-route per turn, so the comparison would have no fixed arm to attribute results to."""
+    re-route per turn, so the comparison would have no fixed arm to attribute results to.
+
+    Resolvability is asked once per team the job samples for, because that is the identity
+    the call carries: a name only one team can reach fails every turn for the other keys,
+    which is the growing error count this check exists to prevent."""
     if llm_router is not None and _is_configured_pre_routing_strategy(llm_router, model):
         raise HTTPException(
             status_code=400,
             detail=f"{field_name} '{model}' is an auto-router; it must be a plain model",
         )
-    if router_resolves_model(llm_router, model):
+    unreachable: Final = tuple(team for team in team_ids if judge_target(llm_router, model, team).via == "nothing")
+    if not unreachable:
         return
-    import litellm
+    raise HTTPException(
+        status_code=400,
+        detail=(
+            f"{field_name} '{model}' is neither a model configured on this proxy nor a "
+            "provider-qualified public model name (e.g. 'anthropic/claude-sonnet-5')" + _for_teams(unreachable)
+        ),
+    )
 
-    try:
-        litellm.get_llm_provider(model=model)
-    except Exception as e:
-        raise HTTPException(
-            status_code=400,
-            detail=(
-                f"{field_name} '{model}' is neither a model configured on this proxy nor a "
-                "provider-qualified public model name (e.g. 'anthropic/claude-sonnet-5')"
-            ),
-        ) from e
+
+def _for_teams(team_ids: Sequence[str | None]) -> str:
+    """Name the teams a fault applies to, when it does not apply to every key alike."""
+    named: Final = tuple(sorted(team for team in team_ids if team is not None))
+    return f" for team {', '.join(named)}" if named else ""
+
+
+_JUDGED_ROLES: Final[frozenset[StrategyRouterDependencyRole]] = frozenset({"tier", "default"})
+
+
+def _router_arm_models(llm_router: "Router | None", router_name: str) -> tuple[tuple[str, str], ...]:
+    """``(role, model_name)`` for every model the router under evaluation can answer with.
+
+    Drawn from ``strategy_router_dependencies``, the single answer to "what does this router
+    call", so this cannot disagree with the health check's reading of the same deployment.
+    Only the roles that SERVE are arms: the classifier and embedding models pick the tier,
+    they never produce a response anyone judges, so a judge sharing them carries no
+    self-preference.
+
+    A semantic auto-router keeps its routes in an opaque config blob or a file, so only its
+    default model is enumerable and the guard below is incomplete for it. That direction is
+    deliberate: it can miss a collision, never invent one.
+
+    Which tiers a router declares is a property of its config and not of who is calling, so
+    this lookup is unscoped; what each tier NAME resolves to is the team-dependent half, and
+    it belongs to the caller that compares them.
+    """
+    deployments: Final = llm_router.get_model_list(model_name=router_name) if llm_router is not None else None
+    return tuple(
+        dict.fromkeys(
+            (dependency.role, dependency.model_name)
+            for deployment in deployments or ()
+            for dependency in strategy_router_dependencies(deployment["litellm_params"])
+            if dependency.role in _JUDGED_ROLES
+        )
+    )
+
+
+def _judge_collisions_for_team(
+    llm_router: "Router | None", data: StartShadowEvalRequest, team_id: str | None
+) -> tuple[tuple[str, str], ...]:
+    """``(role, model_name)`` for each arm the judge would also be, as one team's keys see it.
+
+    Both sides resolve under the SAME team, since two names are the same model only for a
+    caller who can reach both; resolving the judge for one team against an arm for another
+    invents a collision no request could produce.
+    """
+    judge: Final = judge_target(llm_router, data.judge_model, team_id).models
+    return tuple(
+        (role, model)
+        for role, model in (
+            *_router_arm_models(llm_router, data.router_name),
+            *((("baseline", data.baseline_model),) if data.baseline_model is not None else ()),
+        )
+        if judge & judge_target(llm_router, model, team_id).models
+    )
+
+
+def _validate_judge_is_not_a_candidate(
+    llm_router: "Router | None", data: StartShadowEvalRequest, team_ids: Sequence[str | None]
+) -> None:
+    """Reject a judge that is one of the two arms it grades.
+
+    A judge scores its own output higher than a rival's, so a run whose judge also serves an
+    arm reports a win rate for that arm that measures the judge rather than the models, and
+    the whole job's spend buys a result that has to be discarded. Both arms are in scope: the
+    router answers with a tier or default model in either direction, and a reverse job's
+    ``baseline_model`` is the fixed arm the router is compared against.
+
+    Names are compared by what would ANSWER them, not by spelling: the shipped default judge
+    ``anthropic/claude-sonnet-5`` collides with a tier deployment an admin named
+    ``sonnet-tier``, and an alias collides with its target, neither of which a string
+    comparison sees.
+
+    A collision for ONE team is a collision for the job, because the verdicts every key
+    produces land in the same win rates.
+    """
+    collisions: Final = tuple(
+        dict.fromkeys(
+            collision for team_id in team_ids for collision in _judge_collisions_for_team(llm_router, data, team_id)
+        )
+    )
+    if not collisions:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail=(
+            f"judge_model '{data.judge_model}' is also an arm this job would judge: "
+            + ", ".join(f"{role} model '{model}'" for role, model in collisions)
+            + ". A judge scores its own answers higher than a rival's, so the win rates would "
+            "measure the judge; pick a judge that serves neither arm"
+        ),
+    )
 
 
 def _is_unique_violation(error: Exception) -> bool:
@@ -1012,9 +1132,6 @@ async def start_shadow_eval(
         raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
     if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name):
         raise HTTPException(status_code=400, detail=f"'{data.router_name}' is not a configured auto-router")
-    _validate_plain_model(llm_router, data.judge_model, "judge_model")
-    if data.baseline_model is not None:
-        _validate_plain_model(llm_router, data.baseline_model, "baseline_model")
     token_rows: Final = await _verification_tokens(prisma_client).find_many(
         where={"token": {"in": list(data.api_key_ids)}}  # mutable-ok: Prisma filter
     )
@@ -1028,6 +1145,14 @@ async def start_shadow_eval(
             ),
         )
 
+    # Every model check below runs once per team the job samples for, since that is the
+    # identity the shadow and judge calls carry and therefore what the router selects on.
+    team_ids: Final = tuple(dict.fromkeys(row.team_id for row in token_rows or ()))
+    _validate_plain_model(llm_router, data.judge_model, "judge_model", team_ids)
+    if data.baseline_model is not None:
+        _validate_plain_model(llm_router, data.baseline_model, "baseline_model", team_ids)
+    _validate_judge_is_not_a_candidate(llm_router, data, team_ids)
+
     # A job whose window passed or whose budget ran out stopped sampling on its own,
     # but its legs still hold their slots in the per-key, per-direction partial unique index
     # until stamped; free them so a new eval can start. Sweeping both directions is deliberate.

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -94,6 +94,9 @@ class DailySpendRecord(Protocol):
     @property
     def prompt_caching_savings_spend(self) -> float: ...
 
+    @property
+    def gateway_injected_caching_savings_spend(self) -> float: ...
+
     @property
     def autorouter_savings_spend(self) -> float: ...
 
@@ -137,6 +140,7 @@ class _GroupingSetsRow(SimpleNamespace):
     compression_saved_tokens: int | None
     compression_savings_spend: float | None
     prompt_caching_savings_spend: float | None
+    gateway_injected_caching_savings_spend: float | None
     autorouter_savings_spend: float | None
     api_requests: int | None
     successful_requests: int | None
@@ -189,6 +193,9 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) ->
     existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0
     existing_metrics.compression_savings_spend += record.compression_savings_spend or 0
     existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0
+    existing_metrics.gateway_injected_caching_savings_spend += (  # rebind-ok: this accumulator mutates its target in place for every metric on the row
+        record.gateway_injected_caching_savings_spend or 0
+    )
     existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0
     existing_metrics.api_requests += record.api_requests or 0
     existing_metrics.successful_requests += record.successful_requests or 0
@@ -721,6 +728,7 @@ def _build_aggregated_sql_query(
             SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
             SUM(compression_savings_spend)::float AS compression_savings_spend,
             SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
+            SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend,
             SUM(autorouter_savings_spend)::float AS autorouter_savings_spend,
             SUM(api_requests)::bigint AS api_requests,
             SUM(successful_requests)::bigint AS successful_requests,
@@ -799,6 +807,7 @@ def _build_entity_rollup_sql_query(
             SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
             SUM(compression_savings_spend)::float AS compression_savings_spend,
             SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
+            SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend,
             SUM(autorouter_savings_spend)::float AS autorouter_savings_spend,
             SUM(api_requests)::bigint AS api_requests,
             SUM(successful_requests)::bigint AS successful_requests,
@@ -934,6 +943,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics:
         compression_saved_tokens=record.compression_saved_tokens or 0,
         compression_savings_spend=record.compression_savings_spend or 0,
         prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0,
+        gateway_injected_caching_savings_spend=record.gateway_injected_caching_savings_spend or 0,
         autorouter_savings_spend=record.autorouter_savings_spend or 0,
         api_requests=record.api_requests or 0,
         successful_requests=record.successful_requests or 0,
@@ -1200,6 +1210,7 @@ async def get_daily_activity(
                 total_compression_saved_tokens=metadata_metrics.compression_saved_tokens,
                 total_compression_savings_spend=metadata_metrics.compression_savings_spend,
                 total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend,
+                total_gateway_injected_caching_savings_spend=metadata_metrics.gateway_injected_caching_savings_spend,
                 total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend,
                 page=page,
                 total_pages=-(-total_count // page_size),  # Ceiling division
@@ -1372,6 +1383,9 @@ async def get_daily_activity_aggregated(
                 total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens,
                 total_compression_savings_spend=aggregated["totals"].compression_savings_spend,
                 total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend,
+                total_gateway_injected_caching_savings_spend=aggregated[
+                    "totals"
+                ].gateway_injected_caching_savings_spend,
                 total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend,
                 page=1,
                 total_pages=1,

litellm/proxy/management_endpoints/mcp_management_endpoints.py

@@ -204,6 +204,7 @@ if MCP_AVAILABLE:
         MCP_ADMIN_CONFIG_CREDENTIAL_KEYS,
         MCPAuth,
         MCPCredentials,
+        normalize_upstream_header_name,
     )
     from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
@@ -239,9 +240,26 @@ if MCP_AVAILABLE:
                 detail={"error": error_messages_text},
             )
 
+    def _validate_upstream_token_header(payload: McpServerPayloadLike) -> None:
+        credentials: Final = getattr(payload, "credentials", None)
+        raw: Final = credentials.get("upstream_token_header") if isinstance(credentials, dict) else None
+        if not isinstance(raw, str) or raw == "":
+            return
+        if normalize_upstream_header_name(raw) is None:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail={
+                    "error": (
+                        f"Invalid upstream_token_header {raw!r}: must be a valid HTTP header name "
+                        "(RFC 7230 token, e.g. 'esb-oauth')"
+                    )
+                },
+            )
+
     def validate_and_normalize_mcp_server_payload(payload: McpServerPayloadLike) -> None:
         _base_validate_and_normalize_mcp_server_payload(payload)
         _validate_mcp_server_name_fields(payload)
+        _validate_upstream_token_header(payload)
 
     def stamp_omitted_oauth2_flow(payload: NewMCPServerRequest) -> None:
         """Fallback only: fill in oauth2_flow when an oauth2 create omits it.
@@ -739,6 +757,7 @@ if MCP_AVAILABLE:
         ("aws_region_name", "aws_region_name"),
         ("aws_service_name", "aws_service_name"),
         ("upstream_resource", "upstream_resource"),
+        ("upstream_token_header", "upstream_token_header"),
     )
 
     def _has_non_admin_config_credentials(credentials: "MCPCredentials | None") -> bool:

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -85,6 +85,8 @@ from litellm.router_strategy.complexity_router import (
 )
 from litellm.router_utils.auto_router_model_naming import (
     STRATEGY_ROUTER_PARAM_FIELDS,
+    carries_complexity_router_settings,
+    validate_complexity_router_config_placement,
     validate_complexity_router_config_write,
     validate_strategy_router_model_write,
 )
@@ -226,14 +228,19 @@ def _strategy_router_write_violation(
     )
     if config_violation is not None:
         return config_violation
-    if incoming_params.model is None:
-        return None
     present_fields: Final = frozenset(
         field
         for field in STRATEGY_ROUTER_PARAM_FIELDS
         for source in (incoming_params, existing_params)
         if source is not None and getattr(source, field, None) is not None
     )
+    # Scope reads the incoming model because the stored one is encrypted at rest.
+    if carries_complexity_router_settings(incoming_params.model, present_fields):
+        placement_violation: Final = validate_complexity_router_config_placement(incoming_params.model_extra)
+        if placement_violation is not None:
+            return placement_violation
+    if incoming_params.model is None:
+        return None
     return validate_strategy_router_model_write(model=incoming_params.model, present_fields=present_fields)
 
 

litellm/proxy/proxy_server.py

@@ -40,7 +40,7 @@ import anyio
 import websockets
 import websockets.exceptions
 from pydantic import BaseModel, Json, JsonValue
-from typing_extensions import NotRequired, assert_never
+from typing_extensions import NotRequired, ReadOnly, assert_never
 
 from litellm._uuid import uuid
 from litellm.constants import (
@@ -116,6 +116,11 @@ from litellm.router_utils.add_retry_fallback_headers import (
     get_fallback_errors_from_headers,
     get_hidden_params_dict,
 )
+from litellm.router_utils.auto_router_model_naming import (
+    STRATEGY_ROUTER_PARAM_FIELDS,
+    carries_complexity_router_settings,
+    validate_complexity_router_config_placement,
+)
 from litellm.types.utils import (
     ModelResponse,
     ModelResponseStream,
@@ -290,6 +295,7 @@ from litellm.proxy.auth.auth_utils import (
     is_request_body_safe,
     warn_once_if_custom_auth_skips_common_checks,
 )
+from litellm.proxy.auth.fallback_model_access import router_fallback_access_check
 from litellm.proxy.auth.handle_jwt import JWTHandler
 from litellm.proxy.auth.litellm_license import LicenseCheck
 from litellm.proxy.auth.model_checks import (
@@ -381,6 +387,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
 from litellm.proxy.config_resolvers import resolve_fields
 from litellm.proxy.config_resolvers.alerting import (
     EMAIL_DESCRIPTORS,
+    MS_TEAMS_DESCRIPTORS,
     SLACK_DESCRIPTORS,
 )
 from litellm.proxy.container_endpoints.endpoints import router as container_router
@@ -4150,6 +4157,28 @@ def validate_deployment_max_agentic_loops(model: Mapping[str, object]) -> None:
     )
 
 
+def validate_deployment_complexity_router_placement(model: Mapping[str, object]) -> None:
+    """
+    Reject a complexity-router setting written one level above `complexity_router_config`.
+
+    Checked here rather than on `LiteLLM_Params` for the same reason as
+    `max_agentic_loops`: the proxy builds its router with
+    `ignore_invalid_deployments=True`, so a rejection further down turns a bad
+    deployment into a silently missing model instead of a refusal to start.
+    """
+    litellm_params: Final = model.get("litellm_params")
+    if not isinstance(litellm_params, Mapping):
+        return
+    present_fields: Final = frozenset(
+        field for field in STRATEGY_ROUTER_PARAM_FIELDS if litellm_params.get(field) is not None
+    )
+    if not carries_complexity_router_settings(str(litellm_params.get("model") or ""), present_fields):
+        return
+    violation: Final = validate_complexity_router_config_placement(litellm_params)
+    if violation is not None:
+        raise ValueError(f"model {model.get('model_name', '')!r}: {violation}")
+
+
 def pin_complexity_router_model_id(model: dict) -> None:  # mutable-ok: out-param, model_info is stamped in place
     """
     Stamps `model_info.id` from the raw litellm_params before plugin resolution swaps
@@ -5498,6 +5527,7 @@ class ProxyConfig:
                     if isinstance(v, str) and v.startswith("os.environ/"):
                         model["litellm_params"][k] = get_secret(v)
                 validate_deployment_max_agentic_loops(model)
+                validate_deployment_complexity_router_placement(model)
                 pin_complexity_router_model_id(model)
                 complexity_router_config = model["litellm_params"].get("complexity_router_config")
                 if isinstance(complexity_router_config, dict):
@@ -5580,6 +5610,7 @@ class ProxyConfig:
                 async_only_mode=True  # only init async clients
             ),
             ignore_invalid_deployments=True,  # don't raise an error if a deployment is invalid
+            fallback_access_check=router_fallback_access_check,
         )
 
         if redis_usage_cache is not None and router.cache.redis_cache is None:
@@ -6039,6 +6070,7 @@ class ProxyConfig:
                         ),
                         search_tools=search_tools,
                         ignore_invalid_deployments=True,
+                        fallback_access_check=router_fallback_access_check,
                     )
                     verbose_proxy_logger.debug("updated llm_router: %s", llm_router)
             else:
@@ -16300,6 +16332,11 @@ def _apply_callback_role_gate(entries: list, is_full_admin: bool) -> list:
     return [{**entry, "variables": _redact_callback_env_vars(entry.get("variables") or {})} for entry in entries]
 
 
+class _AlertingDestinationEntry(TypedDict):
+    name: ReadOnly[str]
+    variables: ReadOnly[Mapping[str, str | None]]
+
+
 def _apply_alerting_env_role_gate(env_vars: dict, is_full_admin: bool) -> dict:
     if is_full_admin:
         return mask_sensitive_keys(env_vars, _ALERTING_SENSITIVE_VARS)
@@ -16949,6 +16986,17 @@ async def get_config(
             }
         )
 
+        _ms_teams_values, _ = resolve_fields(
+            MS_TEAMS_DESCRIPTORS, environment_variables, os.environ, empty_db_is_set=True
+        )
+        _ms_teams_env_vars: Final = _apply_alerting_env_role_gate(_ms_teams_values, is_full_admin)
+
+        ms_teams_alerting_entry: Final[_AlertingDestinationEntry] = {
+            "name": "ms_teams",
+            "variables": _ms_teams_env_vars,
+        }
+        alerting_data.append(ms_teams_alerting_entry)
+
         if llm_router is None:
             _router_settings = {}
         else:
@@ -16958,6 +17006,7 @@ async def get_config(
             "status": "success",
             "callbacks": _data_to_return,
             "alerts": alerting_data,
+            "active_alerting_destinations": tuple(_alerting),
             "router_settings": _router_settings,
             "available_callbacks": all_available_callbacks,
         }

litellm/proxy/schema.prisma

@@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)

litellm/proxy/spend_tracking/savings.py

@@ -15,6 +15,10 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
 from litellm.litellm_core_utils.llm_cost_calc.utils import _get_cost_per_unit, generic_cost_per_token
+from litellm.types.integrations.anthropic_cache_control_hook import (
+    GATEWAY_INJECTED_CACHE_METADATA_KEY,
+    GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
+)
 
 if TYPE_CHECKING:
     from litellm.router import Router
@@ -25,6 +29,7 @@ class SavingsSpend(NamedTuple):
     compression: float
     prompt_caching: float
     autorouter: float = 0.0
+    gateway_injected_caching: float = 0.0
 
 
 def _input_cache_read_and_write_cost(info: ModelInfo | None) -> tuple[float, float, float]:
@@ -391,6 +396,28 @@ def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage |
         return None
 
 
+def marks_gateway_injection(metadata: Mapping[str, object] | None, model_id: str | None) -> bool:
+    """Whether the gateway put cache breakpoints on the payload THIS row was billed for.
+
+    ``AnthropicCacheControlHook.record_gateway_injection`` stamps the deployment it
+    injected for, and a row carries the deployment it was billed for, so the two agree
+    only on the leg that was actually injected. Every retry, failover and fallback of a
+    request shares one metadata bucket and one ``litellm_call_id``, so the deployment is
+    what tells those legs apart, and a marker left by a sibling reads here as no injection
+    without anyone having to strip it. An injection that ran before any deployment was
+    chosen is in the payload every leg sends, so it is marked for all of them and credits
+    each. Absent on requests the gateway never acted on
+    (client-supplied ``cache_control``, implicit provider caching) and on rows written
+    before the marker shipped; all of it is the fail-closed direction.
+    """
+    if not metadata:
+        return False
+    injected_deployment: Final = metadata.get(GATEWAY_INJECTED_CACHE_METADATA_KEY)
+    if not isinstance(injected_deployment, str):
+        return False
+    return injected_deployment in (GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT, model_id)
+
+
 def extract_cache_read_tokens(usage_object: Mapping[str, object] | None) -> int:
     """Cache-read tokens from a logged usage object, whatever shape recorded them.
 
@@ -533,6 +560,7 @@ def compute_savings_spend(
     model: str | None,
     custom_llm_provider: str | None,
     compression_saved_tokens: int,
+    gateway_injected_cache: bool,
     routing_decision: Mapping[str, object] | None = None,
     usage_object: Mapping[str, object] | None = None,
     model_id: str | None = None,
@@ -565,7 +593,23 @@ def compute_savings_spend(
     A request that only writes cache and gets no hits therefore reports negative savings,
     which is accurate: it really did cost more than the uncached call would have. The
     daily rollup increments arithmetically, so those rows offset positive ones in the
-    same bucket. Auto-router savings compare the
+    same bucket.
+
+    Caching is reported twice. ``prompt_caching`` is every net dollar caching saved,
+    whoever caused it, which is what a customer means by "what did caching save me".
+    ``gateway_injected_caching`` is the subset the gateway can claim credit for, carrying
+    a value only when ``gateway_injected_cache`` is set, i.e. litellm itself added the
+    ``cache_control`` breakpoints (configured injection points or the auto prompt-caching
+    flag). A client that sent its own breakpoints, and a provider that
+    caches implicitly (OpenAI, Gemini), produce the same usage shape with no gateway
+    action, so they count toward the total and not toward the attributed figure.
+
+    Reporting both rather than gating the one column keeps the customer-facing number
+    stable across the change and leaves attribution a separate question. The attributed
+    figure is normally the smaller of the two, being a subset of the same requests, but
+    not always: a request that only writes cache and never reads it has negative net
+    savings, and dropping such a request from the attributed figure can lift it above
+    the total. Auto-router savings compare the
     served ``model`` against the counterfactual baseline the router recorded on
     its ``routing_decision``, and are zero unless the two differ. That record
     also says whether the conversation was already underway, which is what tells
@@ -602,6 +646,7 @@ def compute_savings_spend(
     read_discount: Final = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0)
     write_premium: Final = max(cache_creation_input_tokens, 0) * (cache_write_cost - input_cost)
     prompt_caching: Final = read_discount - write_premium
+    gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0
 
     # The figure the logging path recorded wins, before the usage gate on purpose: a row
     # whose usage no longer parses still carries the number computed when it did.
@@ -623,4 +668,5 @@ def compute_savings_spend(
         compression=compression,
         prompt_caching=prompt_caching,
         autorouter=0.0 if autorouter is None else autorouter,
+        gateway_injected_caching=gateway_injected_caching,
     )

litellm/proxy/spend_tracking/spend_management_endpoints.py

@@ -158,6 +158,7 @@ class _SessionSpendRow(TypedDict):
     session_total_spend: float
     mcp_tool_call_count: int
     mcp_tool_call_spend: float
+    session_cache_hit_count: ReadOnly[int]
 
 
 class _SpendSumAggregate(TypedDict, total=False):
@@ -4135,7 +4136,8 @@ async def _build_ui_spend_logs_response(
                        )::int AS mcp_tool_call_count,
                        COALESCE(SUM(spend) FILTER (
                            WHERE call_type IN ('call_mcp_tool', 'list_mcp_tools')
-                       ), 0)::double precision AS mcp_tool_call_spend
+                       ), 0)::double precision AS mcp_tool_call_spend,
+                       COUNT(*) FILTER (WHERE LOWER(cache_hit) = 'true')::int AS session_cache_hit_count
                 FROM "LiteLLM_SpendLogs"
                 WHERE session_id = ANY($1::text[])
                   AND api_key = ANY($2::text[])
@@ -4149,6 +4151,7 @@ async def _build_ui_spend_logs_response(
                     "session_total_spend": float(row.get("session_total_spend") or 0.0),
                     "mcp_tool_call_count": int(row.get("mcp_tool_call_count") or 0),
                     "mcp_tool_call_spend": float(row.get("mcp_tool_call_spend") or 0.0),
+                    "session_cache_hit_count": int(row.get("session_cache_hit_count") or 0),
                 }
                 for row in rows
                 if row.get("session_id")
@@ -4171,6 +4174,7 @@ async def _build_ui_spend_logs_response(
                 if session_stats["mcp_tool_call_count"]:
                     row_dict["mcp_tool_call_count"] = session_stats["mcp_tool_call_count"]
                     row_dict["mcp_tool_call_spend"] = session_stats["mcp_tool_call_spend"]
+                row_dict["session_cache_hit_count"] = session_stats["session_cache_hit_count"]
             enriched.append(row_dict)
         response_data: list = enriched
     else:

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -137,6 +137,7 @@ def _get_spend_logs_metadata(
             cost_breakdown=None,
             compression_savings=None,
             autorouter_savings=autorouter_savings,
+            litellm_gateway_injected_cache=None,
             litellm_call_id=litellm_call_id,
         )
     verbose_proxy_logger.debug(

litellm/proxy/utils.py

@@ -765,7 +765,7 @@ class ProxyLogging:
                 alert_type_config=alert_type_config,
             )
 
-            if self.alerting is not None and "slack" in self.alerting:
+            if self.alerting is not None and ("slack" in self.alerting or "ms_teams" in self.alerting):
                 # NOTE: ENSURE we only add callbacks when alerting is on
                 # We should NOT add callbacks when alerting is off
                 if (
@@ -1442,6 +1442,7 @@ class ProxyLogging:
                 prompt_variables=data.pop("prompt_variables", None) or {},
                 prompt_label=data.pop("prompt_label", None) or {},
                 prompt_version=data.pop("prompt_version", None) or {},
+                request_kwargs=data,
             )
 
             data.update(optional_params)
@@ -2236,7 +2237,7 @@ class ProxyLogging:
             # do nothing if alerting is not switched on (unless it's a soft_budget alert with team-specific emails)
             return
 
-        if self.alerting is not None and "slack" in self.alerting:
+        if self.alerting is not None and ("slack" in self.alerting or "ms_teams" in self.alerting):
             if self.slack_alerting_instance is not None:
                 await self.slack_alerting_instance.budget_alerts(
                     type=type,
@@ -2301,17 +2302,17 @@ class ProxyLogging:
                 and isinstance(request_data["metadata"]["alerting_metadata"], dict)
             ):
                 alerting_metadata = request_data["metadata"]["alerting_metadata"]
+        if "slack" in self.alerting or "ms_teams" in self.alerting:
+            await self.slack_alerting_instance.send_alert(
+                message=message,
+                level=level,
+                alert_type=alert_type,
+                user_info=None,
+                alerting_metadata=alerting_metadata,
+                **extra_kwargs,
+            )
         for client in self.alerting:
-            if client == "slack":
-                await self.slack_alerting_instance.send_alert(
-                    message=message,
-                    level=level,
-                    alert_type=alert_type,
-                    user_info=None,
-                    alerting_metadata=alerting_metadata,
-                    **extra_kwargs,
-                )
-            elif client == "sentry":
+            if client == "sentry":
                 if litellm.utils.sentry_sdk_instance is not None:
                     litellm.utils.sentry_sdk_instance.capture_message(formatted_message)
                 else:

litellm/responses/litellm_completion_transformation/transformation.py

@@ -2661,6 +2661,7 @@ class LiteLLMCompletionResponsesConfig:
             optional_output_details: Final[dict[str, int]] = {
                 field: value
                 for field, value in (
+                    ("audio_tokens", getattr(completion_details, "audio_tokens", None)),
                     ("text_tokens", getattr(completion_details, "text_tokens", None)),
                     ("image_tokens", getattr(completion_details, "image_tokens", None)),
                 )

litellm/responses/main.py

@@ -537,6 +537,7 @@ async def aresponses(
                     prompt_variables=prompt_variables,
                     prompt_label=kwargs.get("prompt_label", None),
                     prompt_version=kwargs.get("prompt_version", None),
+                    request_kwargs=kwargs,
                 )
             input = cast(
                 str | ResponseInputParam,
@@ -692,6 +693,7 @@ def _apply_prompt_management_to_responses_call(
                 prompt_variables=prompt_variables,
                 prompt_label=kwargs.get("prompt_label", None),
                 prompt_version=kwargs.get("prompt_version", None),
+                request_kwargs=kwargs,
             )
         input = cast(
             str | ResponseInputParam,

litellm/router.py

@@ -196,6 +196,7 @@ from litellm.types.router import (
     CustomRoutingStrategyBase,
     Deployment,
     DeploymentTypedDict,
+    FallbackAccessCheck,
     GuardrailTypedDict,
     LiteLLM_Params,
     MockRouterTestingParams,
@@ -420,6 +421,36 @@ def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error
     return has_generated_content or not error.is_pre_first_chunk
 
 
+def _anthropic_stream_raised_error_status(error: Exception) -> int | None:
+    raw_status: Final = getattr(error, "status_code", None)
+    if isinstance(raw_status, int):
+        return raw_status
+    if isinstance(raw_status, str) and raw_status.isdigit():
+        return int(raw_status)
+    response_status: Final = getattr(getattr(error, "response", None), "status_code", None)
+    return response_status if isinstance(response_status, int) else None
+
+
+def _anthropic_stream_fallback_error_for_raised(
+    error: Exception, model: str, has_generated_content: bool
+) -> "MidStreamFallbackError | None":
+    """Same gate as a detected SSE error event; None means the raise propagates unchanged."""
+    from litellm.exceptions import MidStreamFallbackError
+
+    if has_generated_content:
+        return None
+    status_code: Final = _anthropic_stream_raised_error_status(error)
+    if status_code is not None and not _is_retriable_anthropic_status(status_code):
+        return None
+    return MidStreamFallbackError(
+        message=str(error),
+        model=model,
+        llm_provider="anthropic",
+        original_exception=error,
+        is_pre_first_chunk=True,
+    )
+
+
 def _anthropic_stream_commits_now(chunk: object, has_generated_content: bool, buffered_chunk_count: int) -> bool:
     """
     Whether `chunk` should make Router._aanthropic_messages_streaming_iterator
@@ -604,6 +635,7 @@ class Router:
         health_check_ignore_transient_errors: bool = False,
         background_health_check_model_groups: Sequence[str] | None = None,
         enable_weighted_failover: bool = False,
+        fallback_access_check: FallbackAccessCheck | None = None,
     ) -> None:
         """
         Initialize the Router class with the given parameters for caching, reliability, and routing strategy.
@@ -640,6 +672,7 @@ class Router:
             deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600.
             ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
             enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False.
+            fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted).
         Returns:
             Router: An instance of the litellm.Router class.
 
@@ -679,6 +712,7 @@ class Router:
 
         self.set_verbose = set_verbose
         self.ignore_invalid_deployments = ignore_invalid_deployments
+        self.fallback_access_check: Final = fallback_access_check
         self.debug_level = debug_level
         self.enable_pre_call_checks = enable_pre_call_checks
         self.enable_tag_filtering = enable_tag_filtering
@@ -3964,6 +3998,7 @@ class Router:
             prompt_id=prompt_id,
             prompt_variables=prompt_variables,
             prompt_label=prompt_label,
+            request_kwargs=kwargs,
         )
 
         # Filter out prompt management specific parameters from data before merging
@@ -5061,14 +5096,15 @@ class Router:
                     yield chunk
                 for buffered_chunk in buffered_lifecycle_chunks:
                     yield buffered_chunk
-            except MidStreamFallbackError as e:
-                if _anthropic_stream_should_decline_fallback(has_generated_content, e):
-                    for buffered_chunk in buffered_lifecycle_chunks:
-                        yield buffered_chunk
-                    if e.original_exception is not None:
-                        raise e.original_exception from e
-                    raise
-                async for item in self._aanthropic_messages_fallback_attempt(e, initial_kwargs, wrapper):
+            except Exception as stream_error:  # noqa: BLE001  # any raised provider error must reach the fallback gate
+                async for item in self._aanthropic_messages_recover_stream_error(
+                    stream_error,
+                    has_generated_content,
+                    buffered_lifecycle_chunks,
+                    model,
+                    initial_kwargs,
+                    wrapper,
+                ):
                     yield item
             finally:
                 with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
@@ -5080,6 +5116,36 @@ class Router:
         wrapper: Final = FallbackAwareAnthropicMessagesStream(stream_with_fallbacks(), source_iterator)
         return wrapper
 
+    async def _aanthropic_messages_recover_stream_error(
+        self,
+        stream_error: Exception,
+        has_generated_content: bool,
+        buffered_lifecycle_chunks: tuple[bytes, ...],
+        model: str,
+        initial_kwargs: dict[str, Any],  # mutable-ok: handed to _aanthropic_messages_fallback_attempt, which mutates it
+        wrapper: "FallbackAwareAnthropicMessagesStream",
+    ) -> AsyncGenerator[bytes, None]:
+        """Turns a source-iterator failure into a fallback attempt or the error reaching the caller."""
+        from litellm.exceptions import MidStreamFallbackError
+
+        if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback(
+            has_generated_content, stream_error
+        ):
+            for buffered_chunk in buffered_lifecycle_chunks:
+                yield buffered_chunk
+            if stream_error.original_exception is not None:
+                raise stream_error.original_exception from stream_error
+            raise stream_error
+        fallback_error: Final = (
+            stream_error
+            if isinstance(stream_error, MidStreamFallbackError)
+            else _anthropic_stream_fallback_error_for_raised(stream_error, model, has_generated_content)
+        )
+        if fallback_error is None:
+            raise stream_error
+        async for item in self._aanthropic_messages_fallback_attempt(fallback_error, initial_kwargs, wrapper):
+            yield item
+
     async def _aanthropic_messages_fallback_attempt(
         self,
         e: "MidStreamFallbackError",
@@ -6982,9 +7048,13 @@ class Router:
             _sibling_metadata_key: Final = (
                 "metadata" if _fallback_metadata_key == "litellm_metadata" else "litellm_metadata"
             )
-            if isinstance(_sibling_metadata := kwargs.get(_sibling_metadata_key), dict):
-                _sibling_metadata.pop("attempted_fallbacks", None)
-                _sibling_metadata.pop("original_model_group", None)
+            if isinstance(_sibling_metadata := kwargs.get(_sibling_metadata_key), dict) and (
+                "attempted_fallbacks" in _sibling_metadata or "original_model_group" in _sibling_metadata
+            ):
+                _scrubbed_sibling_metadata: Final = _sibling_metadata.copy()
+                _scrubbed_sibling_metadata.pop("attempted_fallbacks", None)
+                _scrubbed_sibling_metadata.pop("original_model_group", None)
+                kwargs[_sibling_metadata_key] = _scrubbed_sibling_metadata
             if isinstance(_fallback_metadata := kwargs.get(_fallback_metadata_key), dict):
                 _fallback_metadata["attempted_fallbacks"] = 0
                 if model_group is not None:
@@ -10802,6 +10872,25 @@ class Router:
 
         return returned_models
 
+    def resolved_litellm_models(self, model_name: str, team_id: str | None = None) -> tuple[str, ...]:
+        """The provider model strings `model_name` can actually be served by on this proxy.
+
+        `get_model_list` composes every channel the request path itself uses (exact name,
+        model_group_alias, routing groups, wildcards), so this answers "which models will
+        answer a call to this name" rather than "what did the admin call it": the deployment
+        name is admin-arbitrary, and two names over one provider model are one model.
+
+        Empty when the name resolves to no deployment. That is not the same fact as "the
+        call will fail" - a provider-qualified public name is served by the SDK with no
+        deployment behind it - so the fallback for an empty result is the caller's policy,
+        never this function's.
+        """
+        return tuple(
+            litellm_model
+            for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
+            if isinstance(litellm_model := deployment.get("litellm_params", {}).get("model"), str) and litellm_model
+        )
+
     def _invalidate_model_group_info_cache(self) -> None:
         """Invalidate the cached model group info.
 
@@ -12025,10 +12114,7 @@ class Router:
             resolve_structured_messages,
         )
 
-        deployments: Final = self.get_model_list(model_name=model) or []
-        candidate_models: Final = [
-            d["litellm_params"]["model"] for d in deployments if d.get("litellm_params", {}).get("model")
-        ]
+        candidate_models: Final = list(self.resolved_litellm_models(model))
 
         metadata_key: Final = self._get_metadata_variable_name_from_kwargs(request_kwargs)
         metadata: Final = request_kwargs.setdefault(metadata_key, {})

litellm/router_strategy/complexity_router/complexity_router.py

@@ -48,6 +48,7 @@ from .config import (
     DEFAULT_REASONING_KEYWORDS,
     DEFAULT_SIMPLE_KEYWORDS,
     DEFAULT_TECHNICAL_KEYWORDS,
+    HOUSEKEEPING_ASK_SENTINELS,
     PLAN_MODE_SYSTEM_SENTINELS,
     PLAN_MODE_TAIL_SENTINELS,
     PLAN_MODE_TOOL_NAME,
@@ -679,8 +680,17 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo
     on the floor's premium model after the user exits plan mode; leaving it unpinned means the
     floor re-detects while plan mode lasts and the first ordinary turn classifies and pins as
     if plan mode had never happened.
+
+    A housekeeping call is transient in the same way, and pinning it is the most expensive mistake
+    of the three: an agent names the conversation on its first turn, so the cheapest tier would be
+    the pin every session starts with, and the real work that follows would run there for the whole
+    TTL. It describes what that one call is, never what the session's traffic looks like.
     """
-    return decision is None or decision.get("cause") not in ("default_model_fallback", "plan_mode")
+    return decision is None or decision.get("cause") not in (
+        "default_model_fallback",
+        "plan_mode",
+        "housekeeping",
+    )
 
 
 class DimensionScore:
@@ -720,6 +730,7 @@ class ClassificationOutcome(NamedTuple):
         "reasoning_override",
         "llm_classifier",
         "heuristic_first_short_circuit",
+        "housekeeping",
         "classifier_plugin",
         "classifier_fallback",
         "default_model_fallback",
@@ -1738,12 +1749,20 @@ class ComplexityRouter(CustomLogger):
         user_message: str,
         request_kwargs: dict[str, Any] | None = None,
         hard_floor: ComplexityTier | str | None = None,
+        hard_ceiling: ComplexityTier | str | None = None,
     ) -> str:
         """hard_floor excludes every candidate whose tiers all sit below it, turning this pick's
         soft floors (a distance penalty a high-scoring cheap model can outweigh) into a hard
         minimum for requests that carry one, e.g. the plan-mode floor. classified_tier arrives
         already clamped to the floor, so the cold-start pool and the classified_tier eligibility
-        mode satisfy it by construction; only the "all" eligibility mode can reach below."""
+        mode satisfy it by construction; only the "all" eligibility mode can reach below.
+
+        hard_ceiling is the same bound in the other direction, for a request whose tier was decided
+        by what it IS rather than by how hard it is: a housekeeping call is placed at the cheapest
+        tier because that is all it is worth, so a bandit trading cost for quality has nothing to
+        win and must not reach above it. Without it the distance penalty is the only thing holding
+        the tier, and a deployment that lowers tier_distance_penalty silently gets the expensive
+        model back while the routing decision still reads as the cheapest tier."""
         from litellm.router_strategy.adaptive_router.bandit import (
             normalized_cost,
             thompson_sample,
@@ -1799,6 +1818,7 @@ class ComplexityRouter(CustomLogger):
         penalty_weight: Final = self.config.tier_distance_penalty
 
         floor_severity: Final = self._active_tier_severity(hard_floor) if hard_floor is not None else None
+        ceiling_severity: Final = self._active_tier_severity(hard_ceiling) if hard_ceiling is not None else None
         best_model: str | None = None
         best_score = float("-inf")
         candidate_scores: Final[list[dict[str, Any]]] = []
@@ -1808,6 +1828,11 @@ class ComplexityRouter(CustomLogger):
                 for model_tier in self._model_tiers.get(model, (classified_tier,))
             ):
                 continue
+            if ceiling_severity is not None and all(
+                self._active_tier_severity(model_tier) > ceiling_severity
+                for model_tier in self._model_tiers.get(model, (classified_tier,))
+            ):
+                continue
             cell = adaptive._cells[(request_type, model)]
             quality_sample = thompson_sample(cell)
             cost_score = normalized_cost(adaptive.model_to_cost.get(model, 0.0), all_costs)
@@ -1881,6 +1906,44 @@ class ComplexityRouter(CustomLogger):
             self._reminder_markers,
         )
 
+    def _matched_housekeeping_sentinel(self, newest_ask: str | None) -> str | None:
+        """The client housekeeping sentinel on this request's newest ask, or None.
+
+        Read from the newest ask alone, never the whole history, for the reason `_newest_turn_ask`
+        exists: a title request quoted into a later turn's context would otherwise keep matching and
+        route real work to the cheapest tier for the rest of the session.
+
+        Declines whenever an operator's classifier plugin owns the decision. The sentinels are
+        caller-controlled text, and displacing the built-in classifier with them only ever spends
+        less; displacing a plugin is different in kind, because a plugin is where an operator
+        encodes policy the tier ladder does not express, so a caller pasting a title prompt could
+        route a request past a sensitivity or identity rule to a pool that rule would have refused.
+        """
+        if self.config.classifier_type == "custom" or not self.config.route_housekeeping_to_cheapest_tier:
+            return None
+        if not newest_ask:
+            return None
+        return next(
+            (
+                sentinel
+                for sentinel in (*HOUSEKEEPING_ASK_SENTINELS, *(self.config.housekeeping_patterns or ()))
+                if sentinel in newest_ask
+            ),
+            None,
+        )
+
+    def _cheapest_configured_tier(self) -> ComplexityTier | str | None:
+        """The least severe tier that has models, or None when none does.
+
+        Tiers can be declared without a pool, so this cannot assume the first name in the severity
+        order is routable; routing to an empty pool is what `default_fallback` exists to catch.
+        """
+        pools: Final = self._tier_pools()
+        name: Final = next((name for name in self.config.tier_names() if pools.get(name)), None)
+        if name is None:
+            return None
+        return name if self.config.has_custom_tiers else ComplexityTier(name)
+
     def _apply_plan_mode_floor(self, tier: ComplexityTier | str) -> ComplexityTier | str:
         """The higher of the decided tier and the plan-mode floor; identity when the floor is unset."""
         floor: Final = self._resolve_plan_mode_floor()
@@ -2476,8 +2539,14 @@ class ComplexityRouter(CustomLogger):
                 ),
             )
 
-        outcome: Final = await self.aclassify(
-            user_message, system_prompt, request_kwargs, resolved_messages, raw_messages=messages
+        housekeeping_sentinel: Final = self._matched_housekeeping_sentinel(newest_ask)
+        housekeeping_tier: Final = self._cheapest_configured_tier() if housekeeping_sentinel is not None else None
+        outcome: Final = (
+            ClassificationOutcome(tier=housekeeping_tier, score=None, signals=("housekeeping",), cause="housekeeping")
+            if housekeeping_tier is not None
+            else await self.aclassify(
+                user_message, system_prompt, request_kwargs, resolved_messages, raw_messages=messages
+            )
         )
         tier, score, signals = outcome.tier, outcome.score, outcome.signals
         classified_tier: Final = tier
@@ -2533,7 +2602,14 @@ class ComplexityRouter(CustomLogger):
             # has plan_floored False, yet adaptive_eligible="all" scores every model and only
             # penalizes tier distance, so without the floor the bandit could still route below
             # it -- and a floor a bandit can slide under is not a floor.
-            routed_model = self._soft_floor_pick(tier, user_message, request_kwargs, hard_floor=plan_floor)
+            # The ceiling tracks the tier as raised, never the placement it started from: escalation
+            # and the plan-mode floor both move a housekeeping call up, and a ceiling still naming
+            # the cheapest tier would then contradict the floor and bound the pick below the tier
+            # the decision reports.
+            housekeeping_ceiling: Final = tier if outcome.cause == "housekeeping" else None
+            routed_model = self._soft_floor_pick(
+                tier, user_message, request_kwargs, hard_floor=plan_floor, hard_ceiling=housekeeping_ceiling
+            )
             adaptive: Final = self._ensure_adaptive_router()
             if adaptive is not None:
                 kwargs_metadata: Final = request_kwargs.setdefault("metadata", {})
@@ -2582,6 +2658,9 @@ class ComplexityRouter(CustomLogger):
             else signals
         )
         decision_cause: Final[RoutingDecisionCause] = "plan_mode" if plan_floored else outcome.cause
+        decision_keyword: Final = (
+            plan_mode_sentinel if plan_floored else (housekeeping_sentinel if outcome.cause == "housekeeping" else None)
+        )
         return PreRoutingHookResponse(
             model=routed_model,
             messages=messages if has_original_messages else None,
@@ -2593,7 +2672,7 @@ class ComplexityRouter(CustomLogger):
                 tier=classified_pool_tier,
                 score=score,
                 signals=decision_signals,
-                matched_keyword=plan_mode_sentinel if plan_floored else None,
+                matched_keyword=decision_keyword,
                 escalation_keyword=escalation_keyword,
                 escalated=escalated,
                 classifier_model=classifier_model,

litellm/router_strategy/complexity_router/config.py

@@ -321,6 +321,18 @@ PLAN_MODE_TAIL_SENTINELS: Final[tuple[str, ...]] = (
     "Plan mode still active",
 )
 PLAN_MODE_SYSTEM_SENTINELS: Final[tuple[str, ...]] = ('You are currently running in "Plan" mode.',)
+
+# Taken verbatim from classifier payloads captured on a live gateway, 789 calls over one day: the
+# first appears on 17 of them and the second on 2. A coding agent names the conversation by quoting
+# the session and asking for a title, so the ask carries the session's engineering vocabulary while
+# the task is the cheapest one the client performs. Only wording observed on the wire belongs here,
+# never a paraphrase: a sentinel that matches nothing costs a substring scan per request and reads
+# as coverage the router does not have. These are client-owned strings that drift with client
+# releases, so operators extend coverage via housekeeping_patterns rather than editing these.
+HOUSEKEEPING_ASK_SENTINELS: Final[tuple[str, ...]] = (
+    "Write the title in the predominant language of the session",
+    "You are coming up with a succinct title for a coding session",
+)
 PLAN_MODE_TOOL_NAME: Final[str] = "exit_plan_mode"
 
 
@@ -770,6 +782,29 @@ class ComplexityRouterConfig(BaseModel):
             "wording the built-ins don't cover, or after a client release changes its strings."
         ),
     )
+    route_housekeeping_to_cheapest_tier: bool = Field(
+        default=True,
+        description=(
+            "Route a coding agent's own housekeeping calls to the cheapest configured tier "
+            "without classifying them. A client names the conversation by quoting the whole "
+            "session and asking for a title, so the ask reads as the session's engineering work "
+            "and lands on the most expensive tier, which is the reverse of what the call is "
+            "worth. Detection is a literal match against client-owned sentinels on the newest "
+            "ask only, so it cannot fire on an earlier turn, and it never lowers what anyone "
+            "else asked for: a keyword_tier_rule or a session pin still decides instead, and an "
+            "escalation keyword or the plan-mode floor still raises the tier from here. Only the "
+            "classifier is displaced, and its call is skipped, so a matched request costs "
+            "nothing to route. Set false to classify these calls like any other."
+        ),
+    )
+    housekeeping_patterns: tuple[str, ...] | None = Field(
+        default=None,
+        description=(
+            "Additional case-sensitive literal sentinels that mark a request as client "
+            "housekeeping, on top of the built-in conversation-title ones. For clients whose "
+            "wording the built-ins don't cover, or after a client release changes its strings."
+        ),
+    )
 
     # Semantic (embedding) matching for keyword_tier_rules instead of literal text matching
     semantic_keyword_matching: bool = Field(
@@ -939,6 +974,15 @@ class ComplexityRouterConfig(BaseModel):
             return None
         return tuple(stripped for pattern in value if (stripped := pattern.strip()))
 
+    @field_validator("housekeeping_patterns")
+    @classmethod
+    def _normalize_housekeeping_patterns(cls, value: tuple[str, ...] | None) -> tuple[str, ...] | None:
+        """Blank patterns are dropped: an empty string substring-matches every request, which would
+        silently route all traffic to the cheapest tier."""
+        if value is None:
+            return None
+        return tuple(stripped for pattern in value if (stripped := pattern.strip()))
+
     @model_validator(mode="after")
     def _validate_plan_mode_min_tier(self) -> "ComplexityRouterConfig":
         if self.plan_mode_min_tier is None:
@@ -1246,6 +1290,28 @@ class ComplexityRouterConfig(BaseModel):
             )
         return self
 
+    @model_validator(mode="after")
+    def _validate_tier_param_placement(self) -> "ComplexityRouterConfig":
+        """Reject a router setting written into a tier entry's request params.
+
+        A tier entry's ``litellm_params`` are request params for that deployment: the
+        pre-routing hook spreads them onto the outbound call, so a config key placed
+        there configures nothing and reaches the provider as an unknown body field.
+        """
+        misplaced: Final = tuple(
+            f"{tier}.{key}"
+            for tier, entries in self.tier_model_configs.items()
+            for entry in entries
+            for key in sorted(frozenset(entry.litellm_params) & COMPLEXITY_ROUTER_CONFIG_KEYS)
+        )
+        if misplaced:
+            raise ValueError(
+                "tier entries carry complexity_router_config settings in their litellm_params, where the "
+                "router never reads them and the outbound request forwards them to the provider as unknown "
+                f"body fields: {', '.join(misplaced)}. Set these on complexity_router_config itself"
+            )
+        return self
+
     def tier_label(self, tier: ComplexityTier) -> str:
         """Operator-facing display name for a tier, falling back to its canonical name."""
         return self.tier_labels.get(tier, "").strip() or tier.value
@@ -1264,5 +1330,14 @@ class ComplexityRouterConfig(BaseModel):
         )
 
 
+COMPLEXITY_ROUTER_CONFIG_KEYS: Final[frozenset[str]] = frozenset(ComplexityRouterConfig.model_fields)
+"""Every setting name this config owns, derived from the model so a field added later is covered.
+
+These names are disjoint from the OpenAI request params, from ``all_litellm_params``, and from the
+``LiteLLM_Params`` fields, so one of them appearing where a request param belongs is always a
+misplaced setting rather than a parameter the caller meant to send.
+"""
+
+
 # Combined default config
 DEFAULT_COMPLEXITY_CONFIG: Final = ComplexityRouterConfig()

litellm/router_utils/auto_router_model_naming.py

@@ -15,7 +15,10 @@ from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final, Literal, TypeAlias
 
-from litellm.router_strategy.complexity_router.config import LLM_CLASSIFIER_TYPES
+from litellm.router_strategy.complexity_router.config import (
+    COMPLEXITY_ROUTER_CONFIG_KEYS,
+    LLM_CLASSIFIER_TYPES,
+)
 
 AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/"
 
@@ -188,6 +191,47 @@ def validate_complexity_router_config_write(complexity_router_config: Mapping[st
     return None
 
 
+_COMPLEXITY_ROUTER_FIELDS: Final[frozenset[str]] = frozenset(
+    field for group in _REQUIRED_FIELD_GROUPS["complexity"] for field in group
+)
+
+
+def carries_complexity_router_settings(model: str | None, present_fields: frozenset[str]) -> bool:
+    """Whether this deployment configures a complexity router, so is judged on its key set.
+
+    Scoped rather than applied to every deployment because the setting names are only
+    unambiguous in this context: ``embedding_model``, for one, is a legitimate flat param
+    on an s3_vectors vector store. ``present_fields`` carries the same merged view
+    ``validate_strategy_router_model_write`` is judged on, so a router named only by its
+    default model is in scope, and a field added to the table above is covered here for free.
+    """
+    return classify_strategy_router_model(model or "") == "complexity" or bool(
+        present_fields & _COMPLEXITY_ROUTER_FIELDS
+    )
+
+
+def validate_complexity_router_config_placement(litellm_params: Mapping[str, object] | None) -> str | None:
+    """Reject a complexity-router setting written beside ``complexity_router_config``.
+
+    The router reads its settings only from ``litellm_params.complexity_router_config``, so a
+    key one level too high configures nothing. It does not stay inert: the alias-marker
+    forwarding carries every unrecognized ``litellm_params`` key onto the outbound request,
+    where the provider rejects it as an unknown body field, and the deployment then fails
+    every call with an error naming an internal config key. Caller scopes; this judges.
+    """
+    if litellm_params is None:
+        return None
+    misplaced: Final = tuple(sorted(frozenset(litellm_params) & COMPLEXITY_ROUTER_CONFIG_KEYS))
+    if not misplaced:
+        return None
+    return (
+        f"litellm_params sets complexity_router_config settings directly: {', '.join(misplaced)}. "
+        "The router reads these only from complexity_router_config, so there they configure nothing "
+        "and are forwarded to the provider as unknown request params, which rejects the call. "
+        "Move them under complexity_router_config."
+    )
+
+
 def validate_strategy_router_model_write(model: str, present_fields: frozenset[str]) -> str | None:
     """Check that writing ``model`` leaves a deployment the router can load.
 

litellm/router_utils/fallback_event_handlers.py

@@ -263,6 +263,25 @@ def _get_fallback_target_model_group(fallback_entry: str | Mapping[str, object])
     return target if isinstance(target, str) else None
 
 
+async def _is_fallback_target_authorized(
+    litellm_router: LitellmRouter,
+    fallback_entry: str | Mapping[str, object],
+    original_model_group: str,
+    kwargs: Mapping[str, object],
+) -> bool:
+    access_check: Final = litellm_router.fallback_access_check
+    target: Final = _get_fallback_target_model_group(fallback_entry)
+    if access_check is None or target is None or target == original_model_group:
+        return True
+    if await access_check(model=target, request_kwargs=kwargs, llm_router=litellm_router):
+        return True
+    verbose_router_logger.info(
+        "Skipping fallback to model_group = %s: caller is not authorized to call it",
+        mask_sensitive_structure(fallback_entry),
+    )
+    return False
+
+
 def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
     """
     True when the request names a file that only exists under one provider's credentials.
@@ -357,6 +376,8 @@ async def run_async_fallback(
                 original_model_group,
             )
             continue
+        if not await _is_fallback_target_authorized(litellm_router, mg, original_model_group, kwargs):
+            continue
         attempt_key = fallback_attempt_key(mg)
         if attempt_key is not None:
             if attempt_key in attempted:
@@ -375,12 +396,14 @@ async def run_async_fallback(
             elif isinstance(mg, dict):
                 kwargs.update(mg)
             fallback_depth = fallback_depth + 1
-            kwargs[metadata_variable_name] = {
-                "original_model_group": original_model_group,
-                **(kwargs.get(metadata_variable_name) or {}),
-                "model_group": kwargs.get("model", None),
-                "attempted_fallbacks": fallback_depth,
-            }
+            _hop_metadata = dict(kwargs.get(metadata_variable_name) or {})
+            _original_model_group_stamp = _hop_metadata.pop("original_model_group", original_model_group)
+            _hop_metadata.pop("model_group", None)
+            _hop_metadata.pop("attempted_fallbacks", None)
+            _hop_metadata["original_model_group"] = _original_model_group_stamp
+            _hop_metadata["model_group"] = kwargs.get("model", None)
+            _hop_metadata["attempted_fallbacks"] = fallback_depth
+            kwargs[metadata_variable_name] = _hop_metadata
             kwargs["fallback_depth"] = fallback_depth
             kwargs["max_fallbacks"] = max_fallbacks
             kwargs["attempted_targets"] = attempted

litellm/router_utils/reasoning_effort_capability.py

@@ -1,10 +1,13 @@
 """Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts.
 
-The model map's supports_*_reasoning_effort flags are the only signal, and each level's polarity
-mirrors how a request path reads that same flag. medium and high are unconditional for a reasoning
-model. minimal and low are opt-out: openai/chat/gpt_5_transformation.py refuses them only when the
-map says false. xhigh and max are opt-in. none is opt-out everywhere except the azure gpt-5 family,
-whose config raises UnsupportedParamsError without an explicit true.
+An entry that states its levels outright in reasoning_effort_levels is read first and wins
+whole, for a model whose set the per-level flags cannot express: Kimi K3 takes low, high and max,
+and no flag can drop medium because medium has none. Every other entry answers through the
+supports_*_reasoning_effort flags below, whose polarity mirrors how a request path reads that same
+flag. medium and high are unconditional for a reasoning model. minimal and low are opt-out:
+openai/chat/gpt_5_transformation.py refuses them only when the map says false. xhigh and max are
+opt-in. none is opt-out everywhere except the azure gpt-5 family, whose config raises
+UnsupportedParamsError without an explicit true.
 
 xhigh is gated on the request path by the openai and azure gpt-5 configs. max is not gated there at
 all: every entry carrying supports_max_reasoning_effort is Claude-family, and
@@ -41,6 +44,7 @@ _EFFORT_FLAGS: Final = (
     ("xhigh", "supports_xhigh_reasoning_effort"),
     ("max", "supports_max_reasoning_effort"),
 )
+_DECLARED_EFFORTS_KEY: Final = "reasoning_effort_levels"
 _OPT_OUT_EFFORTS: Final = ("minimal", "low")
 _OPT_IN_EFFORTS: Final = ("xhigh", "max")
 _UNCONDITIONAL_EFFORTS: Final = frozenset(("medium", "high"))
@@ -69,6 +73,36 @@ def _declared_effort_flags(model_info: Mapping[str, object]) -> Mapping[str, obj
     )
 
 
+def declared_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, ...] | None:
+    """The entry's own answer, read through the same bare twin as the flags so both spellings of one
+    model agree. Present-and-a-list IS the answer, so a declared [] correctly empties the group and
+    an unknown level is dropped rather than raised: the bundled map is enum-validated by
+    validate-model-prices-json, but an operator can put this key on a config.yaml model_info block
+    where that schema never runs, and one mistyped level must not fail every sibling on the proxy."""
+    own: Final = model_info.get(_DECLARED_EFFORTS_KEY)
+    raw: Final = own if own is not None else _bare_model_entry(model_info).get(_DECLARED_EFFORTS_KEY)
+    if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)):
+        return None
+    declared: Final = frozenset(effort for effort in raw if isinstance(effort, str))
+    return tuple(effort for effort in REASONING_EFFORT_ADVERTISEMENT_ORDER if effort in declared)
+
+
+def declared_reasoning_efforts_for_model(model: str, custom_llm_provider: str) -> tuple[str, ...] | None:
+    """The levels an entry declares, resolved from the model string a provider config holds rather
+    than from a router deployment's model_info.
+
+    None means the map has no opinion, either because the entry declares nothing or because it
+    describes no such model, so a caller keeps whatever it did before the entry was described. The
+    entry is read straight off the map rather than through get_model_info, which raises for a model
+    it does not know: a provider config runs on the request path for every model it serves, most of
+    which the map never named, and a lookup miss there must not fail the call.
+    """
+    entry: Final = litellm.model_cost.get(f"{custom_llm_provider}/{model}") or litellm.model_cost.get(model)
+    if not isinstance(entry, dict):
+        return None
+    return declared_reasoning_efforts(entry)
+
+
 def _supports_none_reasoning_effort(model_info: Mapping[str, object], flag: object) -> bool:
     """Opt-in only where a request path refuses the level. AzureOpenAIGPT5Config raises
     UnsupportedParamsError on reasoning_effort='none' without an explicit true, and it is selected
@@ -114,12 +148,25 @@ def resolve_supported_reasoning_efforts(
     unset flag as () would let one custom deployment empty every level its mapped siblings agree
     on. deployment_is_mapped is that provenance, and an operator who wants either answer for an
     off-map deployment gets it by setting supports_reasoning explicitly.
+
+    If supports_reasoning is unset but at least one per-level flag (e.g.
+    supports_minimal_reasoning_effort) is present, treat it as implicitly True, since the
+    per-level flags are evidence the model supports reasoning. An explicit False always wins:
+    it is the operator's escape hatch and must not be overridden by inherited per-level flags.
     """
     supports_reasoning: Final = model_info.get("supports_reasoning")
-    if supports_reasoning is not True:
-        return () if supports_reasoning is False or deployment_is_mapped else None
+    if supports_reasoning is False:
+        return ()
 
     flags: Final = _declared_effort_flags(model_info)
+    has_per_level_flag: Final = any(value is not None for value in flags.values())
+    if supports_reasoning is not True and not has_per_level_flag:
+        return () if deployment_is_mapped else None
+
+    declared: Final = declared_reasoning_efforts(model_info)
+    if declared is not None:
+        return declared
+
     if all(value is None for value in flags.values()):
         return None
 

litellm/types/guardrails.py

@@ -392,6 +392,16 @@ class PresidioConfigModel(PresidioPresidioConfigModelUserInterface):
         default=None,
         description="Path to a JSON file containing ad-hoc recognizers for Presidio",
     )
+    presidio_analyze_chunk_size_bytes: int | None = Field(
+        default=None,
+        description=(
+            "Maximum UTF-8 bytes of text sent in a single Presidio /analyze call. "
+            "Longer texts are split into overlapping chunks of at most this size "
+            "and the merged results are remapped onto the original text. "
+            "Defaults to 500000; set it below your analyzer deployment's request "
+            "body limit, leaving headroom for the rest of the analyze payload."
+        ),
+    )
     mock_redacted_text: dict | None = Field(default=None, description="Mock redacted text for testing")
 
 
@@ -496,6 +506,9 @@ class BedrockGuardrailConfigModel(BaseModel):
     aws_role_name: str | None = Field(default=None, description="AWS role name for assuming roles")
     aws_web_identity_token: str | None = Field(default=None, description="Web identity token for AWS role assumption")
     aws_sts_endpoint: str | None = Field(default=None, description="AWS STS endpoint URL")
+    aws_external_id: str | None = Field(
+        default=None, description="External ID required by the target role's trust policy on sts:AssumeRole"
+    )
     aws_bedrock_runtime_endpoint: str | None = Field(default=None, description="AWS Bedrock runtime endpoint URL")
     checks: BedrockChecksConfigModel | None = Field(
         default=None,
@@ -842,7 +855,7 @@ class BaseLitellmParams(ContentFilterConfigModel):  # works for new and patch up
         default=True,
         description=(
             "Whether to fail the request if the guardrail encounters an error. "
-            "Implemented by guardrail='model_armor' and 'generic_guardrail_api'. "
+            "Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. "
             "True (default) raises the error. False logs a critical error and lets the request proceed, "
             "so only a valid guardrail response can block or modify it."
         ),

litellm/types/integrations/anthropic_cache_control_hook.py

@@ -1,9 +1,14 @@
-from typing import Literal
+from typing import Final, Literal
 
 from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.types.llms.openai import ChatCompletionCachedContent
 
+GATEWAY_INJECTED_CACHE_METADATA_KEY: Final = "litellm_gateway_injected_cache"
+# No deployment had been chosen when the injection happened, so it is in the payload
+# every leg of the request sends. Never a real deployment id.
+GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT: Final = ""
+
 
 class CacheControlMessageInjectionPoint(TypedDict):
     """Type for message-level injection points."""

litellm/types/llms/anthropic.py

@@ -324,7 +324,12 @@ class AnthropicMessagesToolResultParam(TypedDict, total=False):
     is_error: bool
     content: (
         str
-        | Iterable[AnthropicMessagesToolResultContent | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam]
+        | Iterable[
+            AnthropicMessagesToolResultContent
+            | AnthropicMessagesImageParam
+            | AnthropicMessagesDocumentParam
+            | ToolReference
+        ]
     )
     cache_control: dict | ChatCompletionCachedContent | None
 

litellm/types/llms/openai.py

@@ -1,7 +1,7 @@
 from collections.abc import Iterable, Mapping
 from enum import Enum
 from os import PathLike
-from typing import IO, Any, Final, Literal, Optional, Union
+from typing import IO, Any, Final, Literal, Optional, TypeAlias, Union
 
 import httpx
 from openai import Omit
@@ -820,9 +820,21 @@ class ChatCompletionAssistantMessage(OpenAIChatCompletionAssistantMessage, total
     reasoning_items: list[ChatCompletionReasoningItem] | None
 
 
+class ChatCompletionToolReferenceObject(TypedDict):
+    """Anthropic tool-search result block, carried through untouched so it survives a round trip."""
+
+    type: Literal["tool_reference"]  # writable-ok: Pydantic warns on ReadOnly TypedDict fields
+    tool_name: str  # writable-ok: Pydantic warns on ReadOnly TypedDict fields
+
+
+ToolMessageContentPart: TypeAlias = (
+    ChatCompletionTextObject | ChatCompletionImageObject | ChatCompletionToolReferenceObject
+)
+
+
 class ChatCompletionToolMessage(TypedDict):
     role: Literal["tool"]
-    content: str | Iterable[ChatCompletionTextObject | ChatCompletionImageObject]
+    content: str | Iterable[ToolMessageContentPart]  # writable-ok: Pydantic warns on ReadOnly TypedDict fields
     tool_call_id: str
 
 
@@ -1258,6 +1270,8 @@ class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False):
 
 
 class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
+    audio_tokens: int | None = None
+
     reasoning_tokens: int | None = None
 
     text_tokens: int | None = None

litellm/types/management_endpoints/auto_router_endpoints.py

@@ -2,8 +2,9 @@
 Types for auto-router management endpoints
 """
 
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
+from types import MappingProxyType
 from typing import Final, Literal, TypeAlias
 
 from pydantic import BaseModel, Field, computed_field, field_validator, model_validator
@@ -44,9 +45,30 @@ class ComplexityRouterConfigValidationResponse(BaseModel):
 
 
 class AutoRouterRoutingTestRequest(BaseModel):
-    """A single prompt to classify against a complexity-router config that need not be saved yet."""
+    """A single request to classify against a complexity-router config that need not be saved yet.
 
-    prompt: str = Field(description="The prompt to route, as an end user would send it")
+    Carries the same fields the serving path carries, so a dry run classifies what a real turn
+    would classify. `messages`, `system` and `tools` are forwarded to the routing hook untranslated,
+    which is why they are typed loosely: the hook reads whatever dialect the surface produced, and
+    validating them against one surface's schema would reject the others.
+    """
+
+    prompt: str | None = Field(
+        default=None,
+        description="A single ask to route, as an end user would send it. Mutually exclusive with messages",
+    )
+    messages: Sequence[Mapping[str, object]] | None = Field(
+        default=None,
+        description="The full message list to route, exactly as the serving path would receive it. Mutually exclusive with prompt",
+    )
+    system: str | Sequence[Mapping[str, object]] | None = Field(
+        default=None,
+        description="The top-level system prompt an Anthropic /v1/messages body carries beside its messages",
+    )
+    tools: Sequence[Mapping[str, object]] | None = Field(
+        default=None,
+        description="The tool definitions the request advertises, which decide whether the plan-mode floor applies",
+    )
     complexity_router_config: RequestComplexityRouterConfig = Field(
         description="The complexity router config to route against, in the shape /model/new accepts",
     )
@@ -63,13 +85,60 @@ class AutoRouterRoutingTestRequest(BaseModel):
         description="Team the router is being created for. Required for a team admin, who may only test their own team's routers",
     )
 
-    @field_validator("prompt")
+    @field_validator("messages")
     @classmethod
-    def _require_non_blank_prompt(cls, value: str) -> str:
-        if not value.strip():
-            raise ValueError("prompt must not be blank")
+    def _reject_messages_no_surface_accepts(
+        cls, value: Sequence[Mapping[str, object]] | None
+    ) -> Sequence[Mapping[str, object]] | None:
+        """Reject what every supported surface rejects, and nothing beyond it.
+
+        A real request carrying a message with no string role, or with content that is neither text
+        nor a block list, is a 400 on the serving path, so answering it here with a routed tier
+        would promise a decision the request never gets. Only the two keys the dialects agree on
+        are constrained: anything else in a message stays untranslated and unread.
+        """
+        if value is None:
+            return value
+        for index, message in enumerate(value):
+            if not isinstance(role := message.get("role"), str) or not role.strip():
+                raise ValueError(f"messages[{index}] needs a non-empty string role")
+            if (content := message.get("content")) is not None and not isinstance(content, str | list):
+                raise ValueError(f"messages[{index}] content must be a string, a list of blocks, or null")
         return value
 
+    @model_validator(mode="after")
+    def _resolve_request_carrier(self) -> "AutoRouterRoutingTestRequest":
+        if self.prompt is not None and not self.prompt.strip():
+            raise ValueError("prompt must not be blank")
+        if self.messages is not None and not self.messages:
+            raise ValueError("messages must not be empty")
+        if (self.prompt is None) == (self.messages is None):
+            raise ValueError("provide exactly one of prompt or messages")
+        if self.messages is not None:
+            return self
+        return self.model_copy(
+            update={  # mutable-ok: model_copy types update as a plain dict
+                "messages": [  # mutable-ok: the routing hook's signature takes a list of message dicts
+                    {"role": "user", "content": self.prompt}  # mutable-ok: a message is dict-shaped
+                ]
+            }
+        )
+
+    def wire_body(self) -> Mapping[str, object]:
+        """The request kwargs a serving-path request would carry for this body.
+
+        Every value is handed out by identity rather than copied, so the messages the routing hook
+        classifies and the messages its raw-body plan-mode scan reads are one value, as they are on
+        the serving path.
+        """
+        return MappingProxyType(
+            {  # mutable-ok: MappingProxyType needs a dict to wrap
+                key: value
+                for key, value in (("messages", self.messages), ("system", self.system), ("tools", self.tools))
+                if value is not None
+            }
+        )
+
 
 class AutoRouterRoutingTestResponse(BaseModel):
     """Where one prompt would have been routed, and why."""

litellm/types/mcp.py

@@ -1,6 +1,11 @@
 import enum
+import re
+from collections.abc import Awaitable, Callable, Mapping
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal
+from urllib.parse import urlsplit
 
+import httpx
 from pydantic import BaseModel
 from typing_extensions import TypedDict
 
@@ -181,6 +186,15 @@ class MCPCredentials(TypedDict, total=False):
     ``audience``, which is the RFC 8693 token-exchange parameter.
     """
 
+    upstream_token_header: str | None  # writable-ok: pydantic warns it cannot honour ReadOnly here
+    """
+    Which upstream header carries the credential LiteLLM resolves for this server. Omitted when
+    unset, which keeps RFC 6750's default of ``Authorization``. Set it when the upstream expects the
+    gateway's token somewhere else (an ESB terminating its own credential on e.g. ``esb-oauth``), so
+    a separate operator-configured ``Authorization`` reaches the origin untouched. Non-secret, so it
+    is stored in plaintext and returned on admin reads.
+    """
+
     client_private_key: str | None
     """
     PEM private key used to sign the private-key-JWT client_assertion (RFC 7523)
@@ -223,7 +237,92 @@ class MCPCredentials(TypedDict, total=False):
     """
 
 
-MCP_ADMIN_CONFIG_CREDENTIAL_KEYS: Final[tuple[str, ...]] = ("upstream_resource",)
+DEFAULT_CREDENTIAL_HEADER: Final = "Authorization"
+
+_HEADER_NAME_TOKEN: Final = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$")
+
+
+def normalize_upstream_header_name(raw: str) -> str | None:
+    """The trimmed header name if it is a usable RFC 7230 ``token``, else None.
+
+    One owner for the grammar; each caller picks its own failure shape (a config-load raise, an
+    API 400, a typed CredError). An operator-supplied name reaches egress verbatim, so a value
+    carrying CR/LF, spaces or separators must never get that far.
+    """
+    stripped: Final = raw.strip()
+    return stripped if stripped and _HEADER_NAME_TOKEN.match(stripped) else None
+
+
+def same_header(name: str, other: str) -> bool:
+    """Whether two HTTP header names are the same one. They are case-insensitive (RFC 7230 3.2)."""
+    return name.lower() == other.lower()
+
+
+def has_header(headers: Mapping[str, str] | None, name: str) -> bool:
+    """Whether ``headers`` carries ``name`` under any casing."""
+    return bool(headers) and any(same_header(key, name) for key in headers or {})
+
+
+def without_header(headers: Mapping[str, str] | None, name: str) -> dict[str, str] | None:
+    """A copy of ``headers`` with every casing of ``name`` removed, or None if nothing remains.
+
+    The one owner of "drop this credential's header". Both MCP stacks and the upstream-credential
+    resolver share it so a slot can never be dropped case-sensitively in one place and
+    case-insensitively in another, which is how an injected header came to shadow a resolved
+    credential on the v1 path.
+    """
+    if not headers:
+        return None
+    filtered: Final = {key: value for key, value in headers.items() if not same_header(key, name)}
+    return filtered or None
+
+
+_DEFAULT_PORTS: Final[Mapping[str, int]] = MappingProxyType({"http": 80, "https": 443})
+
+
+def crosses_origin(configured: str, target: str) -> bool:
+    """Whether ``target`` leaves ``configured``'s origin, by the rule HTTP clients use.
+
+    Origin is scheme, host and port, not host alone, so a same-host HTTPS downgrade or a port change
+    counts as crossing it. A plain http -> https upgrade of the same host is exempt, matching what
+    httpx exempts when it decides whether to keep ``Authorization`` across a redirect.
+    """
+    a: Final = urlsplit(configured)
+    b: Final = urlsplit(target)
+    port_a: Final = a.port or _DEFAULT_PORTS.get(a.scheme)
+    port_b: Final = b.port or _DEFAULT_PORTS.get(b.scheme)
+    if a.scheme == b.scheme and a.hostname == b.hostname and port_a == port_b:
+        return False
+    return not (
+        a.hostname == b.hostname and a.scheme == "http" and port_a == 80 and b.scheme == "https" and port_b == 443
+    )
+
+
+def custom_credential_slot(headers: Mapping[str, str] | None) -> str | None:
+    """The first header carrying a credential somewhere other than ``Authorization``, if any."""
+    return next((name for name in headers or {} if not same_header(name, DEFAULT_CREDENTIAL_HEADER)), None)
+
+
+def credential_redirect_hook(
+    configured_url: str, slot: str | None
+) -> Callable[[httpx.Request], Awaitable[None]] | None:
+    """An httpx request hook dropping ``slot`` once a redirect leaves ``configured_url``'s origin.
+
+    None when no guard is needed, so callers do not each repeat the exemption: HTTP clients already
+    strip ``Authorization`` across origins, but forward every other header, so only a credential an
+    operator moved to its own slot can be replayed to whatever host the upstream redirects to.
+    """
+    if not configured_url or not slot or same_header(slot, DEFAULT_CREDENTIAL_HEADER):
+        return None
+
+    async def guard(request: httpx.Request) -> None:
+        if slot in request.headers and crosses_origin(configured_url, str(request.url)):
+            del request.headers[slot]
+
+    return guard
+
+
+MCP_ADMIN_CONFIG_CREDENTIAL_KEYS: Final[tuple[str, ...]] = ("upstream_resource", "upstream_token_header")
 """Non-secret credential keys returned on read so the admin form can show and clear them. Mirrors
 ``ADMIN_CONFIG_CREDENTIAL_KEYS`` in ``ui/litellm-dashboard/src/components/mcp_tools/types.tsx``."""
 

litellm/types/mcp_server/mcp_server_manager.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Any, Final, Literal
 
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, field_validator
 
 from litellm.types.mcp import (
     DEFAULT_SUBJECT_TOKEN_TYPE,
@@ -9,6 +9,7 @@ from litellm.types.mcp import (
     MCPAuthType,
     MCPTokenEndpointAuthMethod,
     MCPTransportType,
+    normalize_upstream_header_name,
 )
 
 # MCPInfo now allows arbitrary additional fields for custom metadata
@@ -86,6 +87,22 @@ class MCPServer(BaseModel):
     # today's behavior; "auto" derives the canonical URI from ``url``; any other value is sent
     # verbatim. Resolved by ``oauth_utils.resolve_upstream_resource``.
     upstream_resource: str | None = None
+    # Which upstream header carries the credential LiteLLM resolves for this server (the minted
+    # OAuth token, or the static key). None keeps RFC 6750's default, ``Authorization``. An ESB or
+    # API gateway that terminates its own credential in a private header needs this so a second,
+    # operator-configured ``Authorization`` can pass through to the origin untouched.
+    upstream_token_header: str | None = None
+
+    @field_validator("upstream_token_header")
+    @classmethod
+    def _check_upstream_token_header(cls, value: str | None) -> str | None:
+        if value is None or not value.strip():
+            return None
+        normalized: Final = normalize_upstream_header_name(value)
+        if normalized is None:
+            raise ValueError(f"upstream_token_header must be a valid HTTP header name (RFC 7230 token), got {value!r}")
+        return normalized
+
     # AWS SigV4 fields
     aws_access_key_id: str | None = None
     aws_secret_access_key: str | None = None