Merge remote-tracking branch 'origin/main' into litellm_bedrock_service_tier_drop_params

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

# Conflicts:
#	tests/unit/llms/bedrock/chat/test_converse_transformation.py

.github/workflows/test-unit.yml

@@ -227,6 +227,7 @@ jobs:
               tests/unit/llms/test_oss_decision.py
               tests/unit/llms/test_polling_url_origin_match.py
               tests/unit/llms/test_predibase_transformation.py
+              tests/unit/llms/test_subscription_accounts.py
               tests/unit/llms/a2a
               tests/unit/llms/aiml
               tests/unit/llms/aiohttp_openai

backend/routes/allowlist.py

@@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/cache_settings",
     "/coordination_redis/",
     "/cost_tracking",
+    "/subscription_accounts/",
     "/cost_optimization/",
     "/cost/",
     "/credentials",

litellm-proxy-extras/litellm_proxy_extras/migrations/20261028000000_subscription_accounts/migration.sql

@@ -0,0 +1,21 @@
+SET LOCAL lock_timeout = '10s';
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
+    "subscription_account_id" TEXT NOT NULL,
+    "custom_llm_provider" TEXT NOT NULL,
+    "account_id" TEXT NOT NULL,
+    "label" TEXT,
+    "monthly_fee" DOUBLE PRECISION NOT NULL,
+    "currency" TEXT NOT NULL,
+    "billing_period_start" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT NOT NULL,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_by" TEXT NOT NULL,
+
+    CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
   @@id([event_id, target, version])
   @@index([target, state, next_attempt_at])
 }
+
+model LiteLLM_SubscriptionAccountTable {
+  subscription_account_id String @id @default(uuid())
+  custom_llm_provider String
+  account_id String
+  label String?
+  monthly_fee Float
+  currency String
+  billing_period_start String
+  created_at    DateTime               @default(now()) @map("created_at")
+  created_by String
+  updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")
+  updated_by String
+
+  @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
+}

litellm/constants.py

@@ -68,6 +68,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
         "retry_after",
         "fallbacks",
         "context_window_fallbacks",
+        "content_policy_fallbacks",
         "retry_policy",
         "model_group_retry_policy",
         "model_group_alias",
@@ -420,6 +421,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
     "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
 )
 BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
+REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
+REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0
 
 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
 WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
@@ -897,6 +900,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
     }
 )
 
+SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
 A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
 
 LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [

litellm/integrations/anthropic_cache_control_hook.py

@@ -116,12 +116,12 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection
     return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict))
 
 
-def _served_provider_for_model(model: str) -> str | None:
+def _served_provider_for_model(model: str, api_base: object = None) -> str | None:
     from litellm.exceptions import BadRequestError
     from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 
     try:
-        _, provider, _, _ = get_llm_provider(model=model)
+        _, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None)
     except BadRequestError:
         return None
     return provider
@@ -536,7 +536,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if model is None:
             return False
         hosted_flag: Final = _hosted_openai_dialect_flag(
-            model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider
+            model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
         )
         if hosted_flag is not None:
             return hosted_flag
@@ -559,8 +559,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
 
     @staticmethod
-    def _resolve_provider(model: str) -> str | None:
-        return _served_provider_for_model(model)
+    def _resolve_provider(model: str, api_base: object = None) -> str | None:
+        return _served_provider_for_model(model, api_base)
 
     @staticmethod
     def count_request_cache_breakpoints(

litellm/integrations/otel/model/spans.py

@@ -361,6 +361,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_LensReview",
         "LiteLLM_LensIngestionKey",
         "LiteLLM_LensWorker",
+        "LiteLLM_SubscriptionAccountTable",
         "LiteLLM_LensSignalConfig",
         "LiteLLM_LensTraceSignal",
     )

litellm/litellm_core_utils/core_helpers.py

@@ -982,25 +982,36 @@ def _usage_field(usage: object, name: str) -> object:
     return getattr(usage, name, None)
 
 
-def _cost_number(value: object) -> float | None:
-    if isinstance(value, bool) or not isinstance(value, int | float):
+def finite_cost(value: object) -> float | None:
+    if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value):
         return None
     return float(value)
 
 
+def single_log_line(value: object) -> str:
+    return str(value).replace("\r", "").replace("\n", "")
+
+
+def finite_reported_cost(value: object) -> float | None:
+    cost: Final = finite_cost(value)
+    if cost is None and isinstance(value, float):
+        verbose_logger.warning("Ignoring a non-finite provider-reported cost %s", value)
+    return cost
+
+
 def _reported_cost_value(value: object) -> float | None:
     """A reported cost is a number, or a breakdown whose total lives under ``total_cost``."""
     raw: Final[object] = value
     if isinstance(value, Mapping):
-        return _cost_number(_usage_field(raw, "total_cost"))
-    return _cost_number(value)
+        return finite_reported_cost(_usage_field(raw, "total_cost"))
+    return finite_reported_cost(value)
 
 
 def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None:
     additional_headers: Final[object] = response.hidden_params.get("additional_headers")
     if not isinstance(additional_headers, Mapping):
         return None
-    return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
+    return finite_reported_cost(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
 
 
 def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None:

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
 )
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
 from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
+from litellm.llms.bedrock.common_utils import bedrock_model_reaches_converse_by_default
 from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
 from litellm.types.utils import LlmProviders
 
@@ -938,13 +939,22 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
 )
 
 
-def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
+def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
     original_response: Final = getattr(original_exception, "response", None)
     if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
-        return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
-    if original_response.status_code == 400:
+        return httpx.Response(
+            status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")
+        )
+    if original_response.status_code == status_code:
         return original_response
-    return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
+    return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request)
+
+
+def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> bool:
+    original_response: Final = getattr(original_exception, "response", None)
+    if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
+        return False
+    return original_response.request.url.path.endswith(("/converse", "/converse-stream"))
 
 
 def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
@@ -980,7 +990,7 @@ def _map_bedrock_exception(
                 message=mantle_context_window_message,
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=_bedrock_mantle_context_window_response(original_exception),
+                response=_bedrock_response_with_status(original_exception, 400),
             )
     if (
         "too many tokens" in error_str
@@ -1003,6 +1013,18 @@ def _map_bedrock_exception(
             llm_provider="bedrock",
             response=getattr(original_exception, "response", None),
         )
+    elif (
+        "The provided model identifier is invalid" in error_str
+        and bool(model)
+        and _came_from_bedrock_converse(original_exception)
+        and bedrock_model_reaches_converse_by_default(model)
+    ):
+        raise NotFoundError(
+            message=f"BedrockException - {error_str}",
+            model=model,
+            llm_provider="bedrock",
+            response=_bedrock_response_with_status(original_exception, 404),
+        )
     elif "Malformed input request" in error_str:
         raise BadRequestError(
             message=f"BedrockException - {error_str}",

litellm/litellm_core_utils/get_litellm_params.py

@@ -85,6 +85,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "github_copilot_user_session",
             "fireworks_forward_user_id",
             "responses_api_version",
+            "allowed_openai_params",
             PROVIDER_AFFINITY_HEADER_KWARG_KEY,
         }
     )

litellm/litellm_core_utils/internal_call_metadata.py

@@ -35,6 +35,12 @@ from litellm.types.utils import (
 
 BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
 
+ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved"
+OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved"
+ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key"
+OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key"
+IO_TOKEN_RESERVATION_KEYS: Final = frozenset({ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, ITPM_CACHE_KEY, OTPM_CACHE_KEY})
+
 MODEL_ACCESS_GROUP_METADATA_KEY: Final = "user_api_key_matched_model_access_groups"
 """Where auth records the model access groups that authorized the request, for the spend writer.
 

litellm/litellm_core_utils/litellm_logging.py

@@ -76,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import (
     is_classifier_call,
 )
 from litellm.litellm_core_utils.core_helpers import (
+    finite_cost,
     get_provider_response_headers_from_hidden_params,
     is_expected_client_error,
     native_messages_reported_cost,
@@ -84,6 +85,7 @@ from litellm.litellm_core_utils.core_helpers import (
     proxy_stamped_used_client_oauth_token,
     reconstruct_model_name,
     set_response_cost_in_hidden_params,
+    single_log_line,
 )
 from litellm.litellm_core_utils.error_normalization import normalize_error
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
@@ -568,6 +570,11 @@ def _resolve_mantle_region_for_cost(
 _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
 _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
 _REDACTED_RESULT: Final = TypeAdapter(object)
+_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
+
+
+def _without_guardrail_information(metadata: Mapping[str, object]) -> Mapping[str, object]:
+    return {key: value for key, value in metadata.items() if key != _GUARDRAIL_INFORMATION_KEY}
 
 
 def _provider_response_id(source: object) -> str | None:
@@ -2088,7 +2095,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     litellm_model_name=litellm_model_name,
                     router_model_id=router_model_id or hidden_params.get("model_id"),
                 )
-                return hidden_params["response_cost"]
+                return self._recorded_response_cost(hidden_params["response_cost"])
             elif router_model_id is None and "model_id" in hidden_params:  # use model_id if not already set
                 router_model_id = hidden_params["model_id"]
 
@@ -2188,7 +2195,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 litellm_model_name=litellm_model_name,
                 router_model_id=router_model_id,
             )
-            return total_response_cost
+            return self._recorded_response_cost(total_response_cost)
         except Exception as e:  # error calculating cost
             debug_info = StandardLoggingModelCostFailureDebugInformation(
                 error_str=str(e),
@@ -2214,6 +2221,15 @@ class Logging(LiteLLMLoggingBaseClass):
 
     _response_cost_calculator = response_cost_calculator
 
+    def _recorded_response_cost(self, cost: float | None) -> float | None:
+        if cost is None or finite_cost(cost) is not None:
+            return cost
+        verbose_logger.warning(
+            "Recording a non-finite response_cost as 0 for litellm_call_id=%s", single_log_line(self.litellm_call_id)
+        )
+        self.cost_breakdown = None
+        return 0.0
+
     def uncached_price(self, result: object) -> UncachedPrice:
         previous_cost_breakdown: Final = self.cost_breakdown
         previous_billed_token_rates: Final = self.billed_token_rates
@@ -2566,6 +2582,23 @@ class Logging(LiteLLMLoggingBaseClass):
         attempt.model_call_details = dict(self.model_call_details)
         return attempt
 
+    def guardrail_scoped_copy(self) -> "Logging":
+        request_params: Final = _LITELLM_PARAMS.validate_python(
+            self.litellm_params or {}  # pyright: ignore[reportUnknownMemberType]  # Logging.litellm_params is untyped
+        )
+        litellm_params: Final = {
+            **request_params,
+            **{
+                key: _without_guardrail_information(_METADATA_MAPPING.validate_python(value))
+                for key, value in request_params.items()
+                if key in _PER_CALL_LITELLM_PARAM_DICTS and isinstance(value, dict)
+            },
+        }
+        scoped: Final = copy.copy(self)
+        scoped.litellm_params = litellm_params
+        scoped.model_call_details = {**self.model_call_details, "litellm_params": litellm_params}
+        return scoped
+
     def dispatch_attempt_failure_handlers(
         self, exception: Exception, traceback_exception: str
     ) -> Coroutine[None, None, None]:
@@ -7560,7 +7593,15 @@ def get_standard_logging_object_payload(
         )
         custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
         raw_response_cost: Final = kwargs.get("response_cost")
-        llm_response_cost: Final[float] = raw_response_cost or 0.0
+        numeric_response_cost: Final = raw_response_cost if isinstance(raw_response_cost, int | float) else None
+        priced_response_cost: Final = finite_cost(numeric_response_cost)
+        if numeric_response_cost is not None and priced_response_cost is None:
+            verbose_logger.warning(
+                "Logging a non-finite response_cost=%s as 0 for litellm_call_id=%s",
+                numeric_response_cost,
+                single_log_line(logging_obj.litellm_call_id),
+            )
+        llm_response_cost: Final[float] = priced_response_cost or 0.0
         guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
         finalized_cost: Final = _finalized_cost_of(logging_obj)
         response_cost: Final[float] = (

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
     """
     from litellm.litellm_core_utils.token_counter import get_image_type
 
+    media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
+
     # If we have a valid content type that's not generic, use it
-    if current_content_type and current_content_type not in [
+    if media_type and media_type not in [
         "binary/octet-stream",
         "application/octet-stream",
     ]:
-        return current_content_type
+        return media_type
 
     # Extension to MIME type mapping
     # Supports images, documents, and other common file types

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -912,9 +912,9 @@ def create_anthropic_image_param(
     image_url_input: str | dict,
     format: str | None = None,
     is_bedrock_invoke: bool = False,
-) -> AnthropicMessagesImageParam:
+) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
     """
-    Create an AnthropicMessagesImageParam from an image URL input.
+    Create an Anthropic content block from an image URL input.
 
     Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
     """
@@ -933,14 +933,7 @@ def create_anthropic_image_param(
         if is_bedrock_invoke or image_url.startswith("http://"):
             base64_url: Final = convert_url_to_base64(url=image_url)
             image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
-            return AnthropicMessagesImageParam(
-                type="image",
-                source=AnthropicContentParamSource(
-                    type="base64",
-                    media_type=image_chunk["media_type"],
-                    data=image_chunk["data"],
-                ),
-            )
+            return _anthropic_content_element_factory(image_chunk)
         else:
             # HTTPS URL - pass directly for regular Anthropic
             return AnthropicMessagesImageParam(
@@ -953,14 +946,7 @@ def create_anthropic_image_param(
     else:
         # Convert to base64 for data URIs or other formats
         image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
-        return AnthropicMessagesImageParam(
-            type="image",
-            source=AnthropicContentParamSource(
-                type="base64",
-                media_type=image_chunk["media_type"],
-                data=image_chunk["data"],
-            ),
-        )
+        return _anthropic_content_element_factory(image_chunk)
 
 
 # The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
                         anthropic_content_element=_anthropic_image_param,
                         original_content_element=content,
                     )
-                    anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
+                    anthropic_content_list.append(
+                        cast(
+                            AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
+                            _anthropic_image_param,
+                        )
+                    )
             elif content["type"] == "tool_reference":
                 anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
             elif content["type"] == "file":

litellm/litellm_core_utils/realtime_streaming.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import time
 import traceback
 from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
 from dataclasses import dataclass
@@ -15,13 +16,19 @@ from litellm._logging import redact_internal_details_from_client_message, verbos
 from litellm._uuid import uuid
 from litellm.constants import (
     BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
+    REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
+    REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
     REALTIME_METERED_USAGE_KEY,
     REALTIME_SESSION_FAILURE_LOGGED_KEY,
     REALTIME_SESSION_SUCCESS_LOGGED_KEY,
 )
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
-from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
+from litellm.llms.base_llm.realtime.transformation import (
+    BaseRealtimeConfig,
+    RealtimeBackend,
+    RealtimeSessionUpdateRejection,
+)
 from litellm.types.llms.openai import (
     OpenAIRealtimeEvents,
     OpenAIRealtimeOutputItemDone,
@@ -125,6 +132,65 @@ class _LoggingWorker(Protocol):
 _DECODED_JSON: Final = TypeAdapter(object)
 
 
+_SESSION_FIELDS: Final = TypeAdapter(dict[str, object])
+_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+_SESSION_MERGE_MAX_DEPTH: Final = 8
+
+
+class _SessionUpdateFrame(TypedDict):
+    type: ReadOnly[str]
+    session: ReadOnly[Mapping[str, object]]
+
+
+def _merge_session_fields(
+    base: Mapping[str, object], update: Mapping[str, object], depth: int = 0
+) -> Mapping[str, object]:
+    def merged_value(key: str) -> object:
+        if key not in update:
+            return base[key]
+        value: Final = update[key]
+        base_value: Final = base.get(key)
+        if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping):
+            return _merge_session_fields(
+                _SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1
+            )
+        return update[key]
+
+    return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))})
+
+
+def _session_update_frame(session: Mapping[str, object]) -> str:
+    frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session}
+    return json.dumps(frame, default=_SESSION_FIELDS.validate_python)
+
+
+_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection(
+    code="session_update_too_large",
+    message=(
+        "The session.update frames sent before the first conversation content add up to more than 10 MB, "
+        "so this session.update was not applied"
+    ),
+    param="session",
+)
+
+
+def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, object]:
+    return {
+        "type": "error",
+        "event_id": f"event_{uuid.uuid4()}",
+        "error": {
+            "type": "invalid_request_error",
+            "code": "audio_input_not_screenable",
+            "message": (
+                "A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it "
+                "starts answering it, so audio input was not sent. Send text with conversation.item.create instead"
+            ),
+            "param": None,
+            "event_id": client_event_id if isinstance(client_event_id, str) else None,
+        },
+    }
+
+
 def _decode_json_object(payload: str) -> Mapping[str, object]:
     """Decode a realtime frame into its top-level field mapping."""
     return json.loads(payload)
@@ -243,8 +309,17 @@ class RealTimeStreaming:
         event_normalizer: RealtimeEventNormalizer | None = None,
         usage_meter: RealtimeUsageMeter | None = None,
         logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
+        lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
+        lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
+        clock: Callable[[], float] = time.monotonic,
     ):
         self.websocket: _ClientWebSocket = websocket
+        self._clock: Final = clock
+        self._lazy_setup_deadline: Final = clock() + lazy_setup_deadline_seconds
+        self._lazy_setup_content_grace_seconds: Final = lazy_setup_content_grace_seconds
+        self._held_client_content: tuple[str, ...] = ()
+        self._held_client_content_bytes: int = 0
+        self._held_client_content_deadline: float | None = None
         self._replay_scope: Final = client_websocket_scope(websocket)
         self.backend_ws = backend_ws
         self.logging_obj = logging_obj
@@ -300,6 +375,10 @@ class RealTimeStreaming:
         # Gemini Live rejects a follow-up BidiGenerateContentSetup once any
         # content (realtimeInput / clientContent / toolResponse) has been sent.
         self._content_sent_after_setup: bool = False
+        self._held_session_update: Mapping[str, object] | None = None
+        self._applied_session_fields: Mapping[str, object] | None = None
+        self._held_session_update_bytes: int = 0
+        self._unscreened_audio_refused: bool = False
         # Whether this is a transcription-only session (session.type == "transcription",
         # e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and
         # their input_audio_transcription.completed usage drives duration-based cost.
@@ -337,6 +416,17 @@ class RealTimeStreaming:
         ]
     )
     _CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"])
+    _CLIENT_CONTENT_TYPES = frozenset(
+        [
+            "conversation.item.create",
+            "response.create",
+            "input_audio_buffer.append",
+            "input_audio_buffer.commit",
+            "input_audio_buffer.end",
+        ]
+    )
+    _HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create", "input_audio_buffer.append"])
+    _DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"])
     _AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
         "pcm16": {"type": "audio/pcm", "rate": 24000},
         "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
@@ -377,7 +467,7 @@ class RealTimeStreaming:
             message_obj: dict[str, Any] = cast(dict[str, Any], message)
         else:
             message_obj = cast(dict[str, Any], json.loads(message))
-        self._collect_tool_calls_from_response_done(cast(dict, message_obj))
+        self._collect_tool_calls_from_response_done(message_obj)
         if not self._should_store_message(message_obj):
             return
         try:
@@ -502,7 +592,7 @@ class RealTimeStreaming:
         self.store_message(flush_event)
         self._capture_transcription_usage(flush_event)
 
-    def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
+    def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None:
         """Extract function_call items from response.done events for spend logging."""
         try:
             if event_obj.get("type") != "response.done":
@@ -607,6 +697,167 @@ class RealTimeStreaming:
                 self._usage_meter.observe_client_event(sent_event)
         return True
 
+    def _builds_setup_lazily(self) -> bool:
+        if self.provider_config is None:
+            return False
+        return self.provider_config.builds_setup_from_session_update()
+
+    def _holds_session_updates_until_first_content(self) -> bool:
+        return self.session_configuration_request is None and self._builds_setup_lazily()
+
+    async def _rejected_setup_session_update(self, message: str) -> bool:
+        provider_config: Final = self.provider_config
+        if provider_config is None or not provider_config.builds_setup_from_session_update():
+            return False
+        client_event: Final = _decode_json_object(message)
+        session: Final = client_event.get("session")
+        update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
+        if self._leaves_applied_setup_unchanged(update):
+            await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS)
+            return True
+        rejection: Final = provider_config.setup_session_update_rejection(
+            self.model, update, setup_sent=self.session_configuration_request is not None
+        )
+        if rejection is None:
+            return False
+        await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id"))))
+        return True
+
+    def _held_session_update_message(self) -> str:
+        return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS)
+
+    async def _hold_session_update(self, message: str) -> None:
+        client_event: Final = _decode_json_object(message)
+        if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES:
+            await self.websocket.send_text(
+                json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id")))
+            )
+            return
+        session: Final = client_event.get("session")
+        update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
+        held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
+        self._held_session_update = held
+        self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8"))
+        await self._acknowledge_session_setup(held)
+
+    def _setup_preview(self, session: Mapping[str, object]) -> str | None:
+        if self.provider_config is None:
+            return None
+        preview: Final = self.provider_config.transform_realtime_request(
+            _session_update_frame(session), self.model, None
+        )
+        return next(
+            (frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
+        )
+
+    def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
+        applied: Final = self._applied_session_fields
+        if self.session_configuration_request is None or applied is None:
+            return False
+        applied_setup: Final = self._setup_preview(applied)
+        merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update))
+        return (
+            applied_setup is not None
+            and merged_setup is not None
+            and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
+        )
+
+    async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
+        if self.provider_config is None:
+            return
+        setup: Final = self._setup_preview(session)
+        if setup is None:
+            return
+        acknowledged: Final = self.provider_config.transform_session_created_event(
+            self.model, str(self.logging_obj.litellm_trace_id), setup
+        )
+        if acknowledged is None:
+            return
+        event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"}
+        event_str: Final = json.dumps(event)
+        self.store_message(event_str)
+        await self._send_event_to_client(event, event_str)
+
+    async def _send_held_session_update_as_setup(self) -> None:
+        held: Final = self._held_session_update
+        if held is None:
+            return
+        message: Final = self._held_session_update_message()
+        self._held_session_update = None
+        self._held_session_update_bytes = 0
+        if await self._send_to_backend(message):
+            self._applied_session_fields = held
+            await self._send_held_client_content()
+
+    def _holds_client_content(self, msg_type: str | None, message: str) -> bool:
+        held_types: Final = (
+            RealTimeStreaming._DEFERRED_SETUP_HELD_CONTENT_TYPES
+            if self._uses_deferred_backend_setup()
+            else RealTimeStreaming._HELD_CONTENT_TYPES
+        )
+        return (
+            msg_type in held_types
+            and self._held_session_update is not None
+            and self._holds_session_updates_until_first_content()
+            and not self._pending_messages_until_setup
+            and len(self._held_client_content) < RealTimeStreaming._MAX_BUFFERED_MESSAGES
+            and self._held_client_content_bytes + len(message.encode("utf-8")) <= RealTimeStreaming._MAX_BUFFERED_BYTES
+        )
+
+    def _hold_client_content(self, message: str) -> None:
+        if self._held_client_content_deadline is None:
+            self._held_client_content_deadline = self._clock() + self._lazy_setup_content_grace_seconds
+        self._held_client_content = (*self._held_client_content, message)
+        self._held_client_content_bytes += len(message.encode("utf-8"))
+
+    async def _send_held_client_content(self) -> None:
+        held: Final = self._held_client_content
+        self._held_client_content = ()
+        self._held_client_content_bytes = 0
+        self._held_client_content_deadline = None
+        for message in held:
+            await self._send_to_backend(message)
+
+    def _setup_deadline(self) -> float:
+        content_deadline: Final = self._held_client_content_deadline
+        return (
+            self._lazy_setup_deadline if content_deadline is None else min(self._lazy_setup_deadline, content_deadline)
+        )
+
+    async def _send_setup_before_content(self, msg_type: str | None) -> None:
+        if self._holds_session_updates_until_first_content():
+            await self._send_held_session_update_as_setup()
+        await self._send_default_setup_before_first_client_message(msg_type)
+
+    async def _receive_client_message_before_setup_deadline(self) -> str | None:
+        if not self._holds_session_updates_until_first_content():
+            return await self._receive_client_message()
+        receive: Final = asyncio.ensure_future(self._receive_client_message())
+        try:
+            done, _ = await asyncio.wait((receive,), timeout=self._setup_deadline() - self._clock())
+            if not done:
+                await self._send_setup_before_content(None)
+            return await receive
+        finally:
+            receive.cancel()
+
+    async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None:
+        if self.session_configuration_request is not None or self.provider_config is None:
+            return
+        if msg_type == "session.update" or (
+            self._uses_deferred_backend_setup()
+            and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup)
+        ):
+            return
+        default_setup: Final = self.provider_config.session_configuration_request(self.model)
+        if not default_setup:
+            return
+        setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup)
+        await self.backend_ws.send(setup)
+        self._cache_session_configuration_request(setup)
+        self._applied_session_fields = _NO_SESSION_FIELDS
+        await self._send_held_client_content()
+
     def _enforce_transcription_session_model(self, message: str) -> str:
         """Force client transcription session updates to the authorized model.
 
@@ -851,6 +1102,20 @@ class RealTimeStreaming:
         self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
         await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
 
+    async def _send_blocked_turn_to_backend(self, pre_block_backend_message: str | None, item_id: str | None) -> None:
+        if pre_block_backend_message is None and self._is_transcription_session:
+            return
+        await self._send_setup_before_content("conversation.item.create")
+        # Deliver any caller-supplied backend message FIRST so that
+        # protocol contracts requiring a specific ordering (e.g.
+        # Gemini Live's mandatory ``toolResponse`` after a
+        # ``toolCall``) are honored before the guardrail's own
+        # clientContent / cancel messages are sent.
+        if pre_block_backend_message is not None:
+            await self._send_to_backend(pre_block_backend_message)
+        if not self._is_transcription_session:
+            await self._withdraw_blocked_turn(item_id)
+
     async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
         await self._send_guardrail_cancel()
         if item_id is not None and self._deletes_conversation_items:
@@ -958,6 +1223,21 @@ class RealTimeStreaming:
         """Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
         return not self._is_transcription_session and self._has_audio_transcription_guardrails()
 
+    def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
+        return (
+            msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES
+            and self._builds_setup_lazily()
+            and self._should_disable_vad_auto_response()
+        )
+
+    async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None:
+        if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES:
+            return
+        self._unscreened_audio_refused = True
+        await self.websocket.send_text(
+            json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id")))
+        )
+
     async def run_realtime_guardrails(
         self,
         transcript: str,
@@ -1030,15 +1310,7 @@ class RealTimeStreaming:
                 # Use realtime_violation_message if configured; fall back to guardrail error text.
                 error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
 
-                # Deliver any caller-supplied backend message FIRST so that
-                # protocol contracts requiring a specific ordering (e.g.
-                # Gemini Live's mandatory ``toolResponse`` after a
-                # ``toolCall``) are honored before the guardrail's own
-                # clientContent / cancel messages are sent.
-                if pre_block_backend_message is not None:
-                    await self._send_to_backend(pre_block_backend_message)
-                if not self._is_transcription_session:
-                    await self._withdraw_blocked_turn(item_id)
+                await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
                 # Send the policy violation hint (shows as small gray status text in UI).
                 await self.websocket.send_text(
                     json.dumps(
@@ -1136,6 +1408,9 @@ class RealTimeStreaming:
                                 break
                     finally:
                         self._flushing_pending_messages_until_setup = False
+                if self.session_created_sent_to_client and self._builds_setup_lazily():
+                    await self._maybe_send_guardrail_turn_detection_update()
+                    continue
                 if self.session_created_sent_to_client:
                     # A synthetic session.created (with placeholder defaults) was
                     # already forwarded to the client when we connected.  The
@@ -1295,7 +1570,11 @@ class RealTimeStreaming:
             return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
 
     def _backend_refused_session(self, close: BackendClose) -> bool:
-        return close.code != 1000 and not self._backend_sent_frames
+        return (
+            close.code != 1000
+            and not self._backend_sent_frames
+            and not self._holds_session_updates_until_first_content()
+        )
 
     async def _log_session_end(self, transport_refusal: Exception | None) -> None:
         provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
@@ -1488,7 +1767,7 @@ class RealTimeStreaming:
         client_event: _ClientEventFrame
         try:
             while True:
-                message = await self._receive_client_message()
+                message = await self._receive_client_message_before_setup_deadline()
                 if message is None:
                     return ClientLoopExit.CLIENT_DISCONNECTED
 
@@ -1600,6 +1879,7 @@ class RealTimeStreaming:
                     if (
                         msg_type == "session.update"
                         and self.session_configuration_request is None
+                        and self._held_session_update is None
                         and not self._guardrail_turn_detection_update_sent
                         and self._should_disable_vad_auto_response()
                         and (
@@ -1663,12 +1943,25 @@ class RealTimeStreaming:
                 except (json.JSONDecodeError, AttributeError):
                     pass
 
+                if self._refuses_unscreened_audio(msg_type):
+                    await self._refuse_unscreened_audio(msg_type, message)
+                    continue
+
                 ## LOGGING
                 # Log after any in-place modifications (GA remap, guardrail
                 # turn_detection injection) so audit logs reflect what we
                 # actually forward to the backend.
                 self.store_input(message=message)
 
+                if self._holds_client_content(msg_type, message):
+                    self._hold_client_content(message)
+                    continue
+                if self._held_client_content and msg_type != "session.update":
+                    await self._send_setup_before_content(msg_type)
+
+                if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup():
+                    await self._send_setup_before_content(msg_type)
+
                 if self._should_buffer_client_message_until_setup(message):
                     self._buffer_pending_message_until_setup(message)
                     continue
@@ -1697,6 +1990,15 @@ class RealTimeStreaming:
                 # ``_maybe_send_guardrail_turn_detection_update`` would retry.
                 if msg_type == "session.update" and self.provider_config is None:
                     await self._send_session_gate_once()
+                if msg_type == "session.update" and await self._rejected_setup_session_update(message):
+                    continue
+                if msg_type == "session.update" and self._holds_session_updates_until_first_content():
+                    await self._hold_session_update(message)
+                    if self._pending_messages_until_setup:
+                        await self._send_held_session_update_as_setup()
+                    continue
+                if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES:
+                    await self._send_setup_before_content(msg_type)
                 try:
                     sent = await self._send_to_backend(message)
                 except RealtimeTranscriptionProtocolError as e:

litellm/litellm_core_utils/reasoning_effort_utils.py

@@ -1,6 +1,12 @@
+import reprlib
+from collections.abc import Collection, Mapping
+from dataclasses import dataclass
 from typing import Final, Literal
 
+from pydantic import TypeAdapter
+
 import litellm
+from litellm._logging import verbose_logger
 from litellm.constants import (
     DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
     DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
@@ -11,6 +17,54 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
 
 OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"]
 
+_DROP_WARNING: Final = "Dropping %s=%s for model=%r because %s"
+_DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%s for model=%r (drop_params=True) because %s"
+_NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level."
+_NON_STRING_EFFORT_REASON: Final = (
+    "it is not a string. Send reasoning_effort as a level name such as 'high' to set a reasoning level."
+)
+_REASONING_PARAMS: Final = ("reasoning", "reasoning_effort", "thinking")
+OPAQUE_VALUE: Final = TypeAdapter(object)
+
+
+def warn_dropped_reasoning(
+    param: str, value: object, model: str | None, reason: str, *, drop_params_gated: bool
+) -> None:
+    verbose_logger.warning(
+        _DROP_PARAMS_DROP_WARNING if drop_params_gated else _DROP_WARNING, param, reprlib.repr(value), model, reason
+    )
+
+
+def warn_non_string_reasoning_effort(value: object, model: str | None) -> None:
+    effort: Final = as_str_mapping(value)
+    requested_level: Final = effort.get("effort") if effort is not None else value
+    if requested_level is None or isinstance(value, str):
+        return
+    warn_dropped_reasoning("reasoning_effort", value, model, _NON_STRING_EFFORT_REASON, drop_params_gated=False)
+
+
+def warn_dropped_reasoning_params(
+    dropped: Collection[object],
+    request: Mapping[str, object],
+    model: str | None,
+    reason: str,
+    *,
+    drop_params_gated: bool,
+) -> None:
+    for param in _REASONING_PARAMS:
+        if param in dropped and param in request:
+            warn_dropped_reasoning(param, request[param], model, reason, drop_params_gated=drop_params_gated)
+
+
+@dataclass(frozen=True, slots=True)
+class DroppedThinkingBudget:
+    budget_tokens: object
+
+    def warn(self, model: str) -> None:
+        warn_dropped_reasoning(
+            "thinking.budget_tokens", self.budget_tokens, model, _NON_NUMERIC_BUDGET_REASON, drop_params_gated=True
+        )
+
 
 def reasoning_effort_from_thinking_budget(
     budget_tokens: float,
@@ -29,7 +83,9 @@ def reasoning_effort_from_thinking_budget(
     return "minimal"
 
 
-def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> OpenAIStyleReasoningEffort | None:
+def _effort_or_dropped_budget(
+    thinking: object, *, drop_params: bool
+) -> OpenAIStyleReasoningEffort | DroppedThinkingBudget | None:
     fields: Final = as_str_mapping(thinking)
     if fields is None or fields.get("type") != "enabled":
         return None
@@ -37,10 +93,27 @@ def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> Op
     if isinstance(budget, (int, float)) and not isinstance(budget, bool):
         return reasoning_effort_from_thinking_budget(budget)
     if drop_params or litellm.drop_params is True:
-        return None
+        return DroppedThinkingBudget(budget)
     raise non_numeric_thinking_budget_error(budget)
 
 
+def reasoning_effort_from_thinking(
+    thinking: object, *, model: str, drop_params: bool
+) -> OpenAIStyleReasoningEffort | None:
+    effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params)
+    if not isinstance(effort, DroppedThinkingBudget):
+        return effort
+    effort.warn(model)
+    return None
+
+
+def reasoning_effort_from_thinking_silently(
+    thinking: object, *, drop_params: bool
+) -> OpenAIStyleReasoningEffort | None:
+    effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params)
+    return None if isinstance(effort, DroppedThinkingBudget) else effort
+
+
 def non_numeric_thinking_budget_error(budget: object) -> UnsupportedParamsError:
     return UnsupportedParamsError(
         message=(

litellm/litellm_core_utils/safe_json_dumps.py

@@ -1,4 +1,5 @@
 import json
+import math
 from collections.abc import Callable
 from typing import Any, Final
 
@@ -14,16 +15,23 @@ def strip_null_bytes(value: str) -> str:
     return value.replace("\x00", "")
 
 
+def _finite_or_none(value: float | None) -> float | None:
+    return None if isinstance(value, float) and not math.isfinite(value) else value
+
+
 def safe_json_structure(
     data: object,
     max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
     value_transform: Callable[[str | None, str], str] | None = None,
     key: str | None = None,
+    *,
+    non_finite_as_null: bool = False,
 ) -> object:
     """
     Rebuild data out of JSON-native pieces while detecting circular references.
     If a circular reference is detected then a marker string is returned.
-    NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors.
+    NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. With
+    non_finite_as_null, NaN and infinite floats become null, since jsonb rejects them.
 
     value_transform, when given, is applied to every string leaf (and to the
     str() fallback for non-serializable objects) with the mapping key the leaf
@@ -43,7 +51,7 @@ def safe_json_structure(
             cleaned = obj.replace("\x00", "") if "\x00" in obj else obj
             return _transform(key, cleaned)
         if isinstance(obj, (int, float, bool, type(None))):
-            return obj
+            return _finite_or_none(obj) if non_finite_as_null else obj
         # Check for circular reference.
         if id(obj) in seen:
             return "CircularReference Detected"
@@ -65,8 +73,9 @@ def safe_json_structure(
             seen.remove(id(obj))
             return tuple_result
         elif isinstance(obj, set):
-            sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
-            set_result: Final = sorted(sortable_values)
+            members: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
+            present: Final = sorted(member for member in members if member is not None)
+            set_result: Final = present + [None] * members.count(None)
             seen.remove(id(obj))
             return set_result
         elif isinstance(obj, BaseModel):
@@ -88,6 +97,10 @@ def safe_dumps(
     data: object,
     max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
     value_transform: Callable[[str | None, str], str] | None = None,
+    *,
+    non_finite_as_null: bool = False,
 ) -> str:
     """Serialize data to JSON text through safe_json_structure."""
-    return json.dumps(safe_json_structure(data, max_depth, value_transform), default=str)
+    return json.dumps(
+        safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str
+    )

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -15,6 +15,7 @@ from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import (
     PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS,
     chunk_usage,
+    finite_cost,
     provider_reported_usage_cost,
     stamp_litellm_bill_on_usage,
 )
@@ -1014,7 +1015,9 @@ class ChunkProcessor:
                     prompt_tokens_details, cache_creation_token_details
                 )
 
-                if usage_chunk_dict["cost"] is not None:
+                if Usage.dropped_non_finite_cost(usage_chunk):
+                    cost = None
+                elif usage_chunk_dict["cost"] is not None:
                     cost = usage_chunk_dict["cost"]
 
         prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details)
@@ -1444,8 +1447,8 @@ def _set_stream_builder_response_cost(
 def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
     if logging_obj is None:
         return
-    if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance(
-        getattr(usage, "cost", None), (int, float)
+    if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and (
+        finite_cost(getattr(usage, "cost", None)) is not None
     ):
         return
     computed_cost: Final = logging_obj.response_cost_calculator(result=response)

litellm/litellm_core_utils/streaming_handler.py

@@ -40,6 +40,7 @@ from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import (
+    CHOICE_LOGPROBS_OR_JSON,
     CacheCreationTokenDetails,
     CompletionTokensDetailsWrapper,
     Delta,
@@ -1201,7 +1202,10 @@ class CustomStreamWrapper:
                                         choice_json.pop(
                                             "finish_reason", None
                                         )  # for mistral etc. which return a value in their last chunk (not-openai compatible).
-                                        choices.append(StreamingChoices(**choice_json))
+                                        choice_json.pop("logprobs", None)
+                                        choices.append(
+                                            StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None))
+                                        )
                                 except Exception:
                                     choices.append(StreamingChoices())
                             setattr(model_response, "choices", choices)
@@ -1605,7 +1609,7 @@ class CustomStreamWrapper:
                     model_response.system_fingerprint = response_obj["original_chunk"].system_fingerprint
                     self.system_fingerprint = response_obj["original_chunk"].system_fingerprint
             if response_obj["logprobs"] is not None:
-                model_response.choices[0].logprobs = response_obj["logprobs"]
+                model_response.choices[0].logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(response_obj["logprobs"])
 
             if response_obj["usage"] is not None:
                 if isinstance(response_obj["usage"], dict):
@@ -2482,6 +2486,25 @@ class CustomStreamWrapper:
     def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None:
         self._earlier_rounds_spend = earlier
 
+    def partial_response(self) -> "ModelResponse | TextCompletionResponse | None":
+        partial: Final = (
+            litellm.stream_chunk_builder(
+                chunks=self.chunks,
+                messages=self.messages if isinstance(self.messages, list) else None,
+                logging_obj=self.logging_obj,
+                count_prompt_tokens=self.count_prompt_tokens,
+            )
+            if self.chunks
+            else None
+        )
+        usage: Final[object] = getattr(partial, "usage", None)
+        if partial is None or not isinstance(usage, Usage):
+            return None
+        if self.model:
+            partial.model = self.model
+        backfill_missing_cache_usage_fields(usage)
+        return partial
+
     def _record_partial_usage_for_failure(self) -> None:
         """
         A stream that breaks mid-flight still billed the provider for the chunks
@@ -2491,23 +2514,15 @@ class CustomStreamWrapper:
         moves it into terminal failure usage before opening the next attempt,
         so a recovered attempt logs its own usage.
         """
-        if self.logging_obj is None or not self.chunks:
+        if self.logging_obj is None:
             return
         try:
-            partial_response: Final = litellm.stream_chunk_builder(
-                chunks=self.chunks,
-                messages=self.messages if isinstance(self.messages, list) else None,
-                logging_obj=self.logging_obj,
-                count_prompt_tokens=self.count_prompt_tokens,
-            )
+            partial_response: Final = self.partial_response()
             if partial_response is None:
                 return
-            usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
-            if usage is None:
+            usage: Final[object] = getattr(partial_response, "usage", None)
+            if not isinstance(usage, Usage):
                 return
-            if self.model:
-                partial_response.model = self.model
-            backfill_missing_cache_usage_fields(usage)
             partial_cost: Final = self.logging_obj.response_cost_calculator(result=partial_response) or 0.0
             if self._earlier_rounds_spend is None:
                 self.logging_obj.model_call_details["combined_usage_object"] = usage

litellm/llms/anthropic/chat/transformation.py

@@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     place_mid_conversation_system,
     split_leading_system_run,
 )
+from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
 from litellm.llms.base_llm.base_utils import (
     ResponseFormatSchema,
     as_json_object,
@@ -1951,6 +1952,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 if isinstance(effort_value, dict):
                     effort_value = effort_value.get("effort")
                 if not isinstance(effort_value, str):
+                    warn_non_string_reasoning_effort(OPAQUE_VALUE.validate_python(value), model)
                     continue
                 mapped_thinking = AnthropicConfig._map_reasoning_effort(
                     reasoning_effort=effort_value,

litellm/llms/anthropic/common_utils.py

@@ -37,7 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     message_field,
     parts_of,
 )
-from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error
+from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.anthropic.wif import (
     aget_anthropic_wif_token,
@@ -610,11 +610,11 @@ def _numeric_budget_tokens(budget_tokens: object) -> int | None:
         return None
 
 
-def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | None:
+def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | DroppedThinkingBudget:
     if (numeric := _numeric_budget_tokens(budget_tokens)) is not None:
         return numeric
     if drop_params or litellm.drop_params is True:
-        return None
+        return DroppedThinkingBudget(budget_tokens)
     raise non_numeric_thinking_budget_error(budget_tokens)
 
 
@@ -1097,6 +1097,22 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         custom_llm_provider: str,
         drop_params: bool,
     ) -> None:
+        dropped: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
+            model=model,
+            optional_params=optional_params,
+            custom_llm_provider=custom_llm_provider,
+            drop_params=drop_params,
+        )
+        if dropped is not None:
+            dropped.warn(model)
+
+    @staticmethod
+    def translate_legacy_thinking_for_adaptive_model_silently(
+        model: str,
+        optional_params: MutableMapping[str, object],  # mutable-ok: in-place out-param like the sibling helpers
+        custom_llm_provider: str,
+        drop_params: bool,
+    ) -> DroppedThinkingBudget | None:
         """Translate legacy ``thinking.type=enabled`` to adaptive for the
         adaptive-thinking models that reject it (4.7+ and the 5 families).
         Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the
@@ -1105,12 +1121,12 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         ``output_config.effort`` is never overridden.
         """
         if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
-            return
+            return None
         if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
-            return
+            return None
         thinking: Final = validated_object_mapping(optional_params.get("thinking"))
         if thinking is None or thinking.get("type") != "enabled":
-            return
+            return None
 
         budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params)
         display: Final = thinking.get("display")
@@ -1119,8 +1135,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params
         )
         optional_params["thinking"] = {**carried_keys, "type": "adaptive", **supported_display}
-        if budget_tokens is None:
-            return
+        if isinstance(budget_tokens, DroppedThinkingBudget):
+            return budget_tokens
         effort: Final = AnthropicModelInfo._legacy_budget_to_effort(
             model=model, budget_tokens=budget_tokens, custom_llm_provider=custom_llm_provider
         )
@@ -1129,6 +1145,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             "effort": effort,
             **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})),
         }
+        return None
 
     @staticmethod
     def _legacy_budget_to_effort(model: str, budget_tokens: int, custom_llm_provider: str) -> str:

litellm/llms/anthropic/messages/adapters/handler.py

@@ -647,12 +647,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             request_data["output_config"] = extra_kwargs["output_config"]
 
         custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
+        api_base: Final = extra_kwargs.get("api_base")
         (
             openai_request,
             tool_name_mapping,
         ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
             request_data,
             custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
+            api_base=api_base if isinstance(api_base, str) else None,
             drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")),
         )
 

litellm/llms/anthropic/messages/adapters/transformation.py

@@ -24,6 +24,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
 TOOL_NAME_HASH_LENGTH: Final = 8
 TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1  # 55
 PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
+PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"})
 _COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
 
 
@@ -133,6 +134,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
 )
 from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking,
+    reasoning_effort_from_thinking_silently,
 )
 from litellm.llms.anthropic.common_utils import (
     eager_input_streaming_flag,
@@ -255,7 +257,12 @@ class AnthropicAdapter:
         return result
 
     def translate_completion_input_params_with_tool_mapping(
-        self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True
+        self,
+        kwargs,
+        *,
+        custom_llm_provider: str | None = None,
+        api_base: str | None = None,
+        drop_params: bool = True,
     ) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
         """
         Translate Anthropic request params to OpenAI format, returning tool name mapping.
@@ -289,7 +296,9 @@ class AnthropicAdapter:
         ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
             anthropic_message_request=request_body,
             custom_llm_provider=custom_llm_provider,
+            api_base=api_base,
             drop_params=drop_params,
+            report_reasoning_drops=True,
         )
 
         return translated_body, tool_name_mapping
@@ -659,7 +668,9 @@ class LiteLLMAnthropicMessagesAdapter:
     def translate_anthropic_thinking_to_reasoning_effort(
         thinking: AnthropicThinkingParam,
         *,
+        model: str,
         drop_params: bool = True,
+        report_reasoning_drops: bool = False,
     ) -> str | None:
         """
         Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
@@ -678,7 +689,9 @@ class LiteLLMAnthropicMessagesAdapter:
         if thinking_type == "disabled":
             return "none"
         elif thinking_type == "enabled":
-            return reasoning_effort_from_thinking(thinking, drop_params=drop_params)
+            if report_reasoning_drops:
+                return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
+            return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params)
         elif thinking_type == "adaptive":
             # Adaptive thinking: effort is controlled by output_config.effort,
             # not budget_tokens. Return a default; caller should override with
@@ -745,7 +758,7 @@ class LiteLLMAnthropicMessagesAdapter:
             return {"thinking": thinking}
         else:
             reasoning_effort: Final = LiteLLMAnthropicMessagesAdapter.translate_anthropic_thinking_to_reasoning_effort(
-                thinking
+                thinking, model=model
             )
             if reasoning_effort:
                 return {
@@ -1001,7 +1014,9 @@ class LiteLLMAnthropicMessagesAdapter:
                 )
 
     @staticmethod
-    def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool:
+    def _supports_prompt_cache_key(
+        prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None
+    ) -> bool:
         if not model or not custom_llm_provider:
             return False
         if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND:
@@ -1009,7 +1024,17 @@ class LiteLLMAnthropicMessagesAdapter:
         supported_params: Final = litellm.get_supported_openai_params(
             model=model, custom_llm_provider=custom_llm_provider
         )
-        return "prompt_cache_key" in (supported_params or ())
+        if "prompt_cache_key" not in (supported_params or ()):
+            return False
+        if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG:
+            return True
+        _, _, _, resolved_api_base = litellm.get_llm_provider(
+            model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
+        )
+        mistral_config: Final = litellm.MistralConfig()
+        return mistral_config.takes_prompt_cache_key_value(
+            prompt_cache_key
+        ) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base)
 
     @staticmethod
     def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool:
@@ -1042,6 +1067,7 @@ class LiteLLMAnthropicMessagesAdapter:
         new_kwargs: ChatCompletionRequest,
         *,
         custom_llm_provider: str | None = None,
+        api_base: str | None = None,
     ) -> None:
         """Translate metadata fields from Anthropic request to OpenAI request."""
         if "metadata" in anthropic_message_request:
@@ -1050,7 +1076,7 @@ class LiteLLMAnthropicMessagesAdapter:
                 new_kwargs["user"] = metadata["user_id"]
                 prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"])
                 if prompt_cache_key is not None and self._supports_prompt_cache_key(
-                    anthropic_message_request.get("model"), custom_llm_provider
+                    prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base
                 ):
                     new_kwargs["prompt_cache_key"] = prompt_cache_key
 
@@ -1127,6 +1153,7 @@ class LiteLLMAnthropicMessagesAdapter:
         *,
         custom_llm_provider: str | None = None,
         drop_params: bool = True,
+        report_reasoning_drops: bool = False,
     ) -> None:
         """Translate Anthropic thinking to either thinking or reasoning_effort.
 
@@ -1187,7 +1214,10 @@ class LiteLLMAnthropicMessagesAdapter:
             return
 
         reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort(
-            cast(AnthropicThinkingParam, thinking), drop_params=drop_params
+            cast(AnthropicThinkingParam, thinking),
+            model=model,
+            drop_params=drop_params,
+            report_reasoning_drops=report_reasoning_drops,
         )
         if not reasoning_effort:
             return
@@ -1244,8 +1274,10 @@ class LiteLLMAnthropicMessagesAdapter:
         anthropic_message_request: AnthropicMessagesRequest,
         *,
         custom_llm_provider: str | None = None,
+        api_base: str | None = None,
         preserve_midturn_system: bool = False,
         drop_params: bool = True,
+        report_reasoning_drops: bool = False,
     ) -> tuple[ChatCompletionRequest, dict[str, str]]:
         """
         This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format.
@@ -1282,6 +1314,7 @@ class LiteLLMAnthropicMessagesAdapter:
             anthropic_message_request=anthropic_message_request,
             new_kwargs=new_kwargs,
             custom_llm_provider=custom_llm_provider,
+            api_base=api_base,
         )
         ## CONVERT TOOL CHOICE
         self._translate_tool_choice_to_openai(
@@ -1299,6 +1332,7 @@ class LiteLLMAnthropicMessagesAdapter:
             new_kwargs=new_kwargs,
             custom_llm_provider=custom_llm_provider,
             drop_params=drop_params,
+            report_reasoning_drops=report_reasoning_drops,
         )
         ## CONVERT STOP_SEQUENCES
         self._translate_stop_sequences_to_openai(

litellm/llms/anthropic/messages/native/handler.py

@@ -20,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.exception_mapping_utils import exception_type
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.anthropic.common_utils import (
+    AnthropicModelInfo,
     strip_provider_specific_fields_from_anthropic_messages,
     tools_marked_for_a_tool_config_point,
 )
@@ -36,7 +37,6 @@ from litellm.router_utils.reasoning_effort_capability import (
     keeps_effort,
     model_group_from_metadata,
     resolve_deployment_reasoning_effort,
-    with_adaptive_thinking_as_effort,
 )
 from litellm.types.llms.anthropic import AllAnthropicToolsValues
 from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
@@ -185,9 +185,11 @@ def _with_deployment_output_effort(
         ).value
 
     def translate(source: Mapping[str, object]) -> Mapping[str, object]:
-        return with_adaptive_thinking_as_effort(
-            source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params
+        translated: Final = dict(source)
+        AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
+            model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params
         )
+        return translated
 
     requested_output: Final = as_str_mapping(params.get("output_config"))
     caller_output: Final = None if requested_output is None else resolve(requested_output)

litellm/llms/anthropic/messages/native/transformation.py

@@ -21,6 +21,7 @@ from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
 from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts
 from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards
@@ -103,6 +104,7 @@ _MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
 
 class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     _workload_identity_eligible: ClassVar[bool] = True
+    _report_reasoning_drops: ClassVar[bool] = True
 
     @property
     def custom_llm_provider(self) -> str | None:
@@ -695,12 +697,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         ):
             anthropic_messages_optional_request_params.pop("output_format")  # pyright: ignore[reportUnknownMemberType]  # the request params arrive as a bare dict
 
+        requested_reasoning_effort: Final = OPAQUE_VALUE.validate_python(
+            anthropic_messages_optional_request_params.get("reasoning_effort")  # pyright: ignore[reportUnknownMemberType]  # the request params arrive as a bare dict
+        )
         self._translate_reasoning_effort_to_anthropic(
             model=model,
             optional_params=anthropic_messages_optional_request_params,
             max_tokens=max_tokens,
             custom_llm_provider=self._resolved_provider,
         )
+        if self._report_reasoning_drops:
+            warn_non_string_reasoning_effort(requested_reasoning_effort, model)
 
         AnthropicModelInfo.maybe_drop_disabled_thinking(
             model=model,
@@ -708,12 +715,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             custom_llm_provider=self._resolved_provider,
         )
 
-        AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model(
+        dropped_budget: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently(
             model=model,
             optional_params=anthropic_messages_optional_request_params,
             custom_llm_provider=self._resolved_provider,
             drop_params=getattr(litellm_params, "drop_params", None) is True,
         )
+        if dropped_budget is not None and self._report_reasoning_drops:
+            dropped_budget.warn(model)
 
         self._translate_adaptive_effort_for_non_adaptive_model(
             model=model,

litellm/llms/anthropic/messages/responses_adapters/transformation.py

@@ -534,6 +534,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
         output_config: dict[str, object] | None = None,
         auto_summary_level: str = "detailed",
         *,
+        model: str,
         drop_params: bool = True,
     ) -> dict[str, object] | None:
         """
@@ -555,7 +556,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             if isinstance(output_config, dict) and output_config.get("effort"):
                 effort = output_config["effort"]
         elif thinking_type == "enabled":
-            budget_effort: Final = reasoning_effort_from_thinking(thinking, drop_params=drop_params)
+            budget_effort: Final = reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
             if budget_effort is None:
                 return None
             effort = budget_effort
@@ -667,6 +668,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                 thinking,
                 output_config=cast(dict[str, object] | None, output_config),
                 auto_summary_level=auto_summary_level,
+                model=model,
                 drop_params=drop_params,
             )
             if reasoning:

litellm/llms/anthropic/prompt_cache_prediction.py

@@ -9,7 +9,7 @@ from dataclasses import dataclass, field
 from functools import reduce
 from itertools import accumulate, groupby
 from types import MappingProxyType
-from typing import Annotated, Final, Literal, Protocol, TypeAlias
+from typing import Annotated, ClassVar, Final, Literal, Protocol, TypeAlias
 
 import httpx
 from pydantic import ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError
@@ -81,6 +81,10 @@ _DEPLOYMENT_OPTIONS: Final = frozenset(
 )
 
 
+class _BaselineAnthropicMessagesConfig(AnthropicMessagesConfig):
+    _report_reasoning_drops: ClassVar[bool] = False
+
+
 class _StrictModel(LiteLLMBaseModel):
     model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
 
@@ -661,7 +665,7 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M
             TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()),
             dict(options),
         )
-        config: Final = AnthropicMessagesConfig()
+        config: Final = _BaselineAnthropicMessagesConfig()
         body: Final = config.transform_anthropic_messages_request(
             model=resolved_model,
             messages=strip_provider_specific_fields_from_anthropic_messages(messages),

litellm/llms/azure/responses/transformation.py

@@ -14,6 +14,7 @@ from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
 from litellm.llms.azure.common_utils import BaseAzureLLM
+from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.types.llms.openai import *
 from litellm.types.responses.main import *
@@ -92,6 +93,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
         return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)
 
+    def deployment_names_wrap_openai_models(self) -> bool:
+        return True
+
+    def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
+        return SUMMARY_ONLY_REASONING_FIELD_VALUES if self._is_gpt_5_model(model) else None
+
     def get_supported_openai_params(self, model: str) -> list:
         """
         Azure Responses API does not support context_management (compaction).

litellm/llms/azure_ai/chat/transformation.py

@@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     prompt_cache_breakpoint_for_wire,
     with_prompt_cache_breakpoint,
 )
+from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
 from litellm.llms.azure.common_utils import BaseAzureLLM
 from litellm.llms.azure_ai.common_utils import (
     api_key_header_for_base,
@@ -40,6 +41,14 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
 
+_REQUEST_BODY: Final = TypeAdapter(dict[str, object])
+
+
+def _request_model(request: Mapping[str, object]) -> str | None:
+    model: Final = request.get("model")
+    return model if isinstance(model, str) else None
+
+
 class AzureFoundryErrorStrings(str, enum.Enum):
     SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'"
 
@@ -510,7 +519,13 @@ class AzureAIStudioConfig(OpenAIConfig):
             request_data = self._drop_extra_params_from_request_data(request_data, error_text)
         if "Extra inputs are not permitted" in error_text and self._error_has_tool_level_extra_fields(error_text):
             request_data = self._drop_tool_level_extra_fields(request_data, error_text)
-        data: Final = drop_params_from_unprocessable_entity_error(e=e, data=request_data)
+        data: Final = drop_params_from_unprocessable_entity_error(
+            e=e,
+            data=request_data,
+            model=_request_model(_REQUEST_BODY.validate_python(request_data)),
+            drop_params_gated="Extra inputs are not permitted" in error_text
+            and not self._error_has_tool_level_extra_fields(error_text),
+        )
         return data
 
     def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict:
@@ -526,6 +541,14 @@ class AzureAIStudioConfig(OpenAIConfig):
     def _drop_extra_params_from_request_data(self, request_data: dict, error_text: str) -> dict:
         params_to_drop: Final = self._extract_params_to_drop_from_error_text(error_text)
         if params_to_drop:
+            request: Final = _REQUEST_BODY.validate_python(request_data)
+            warn_dropped_reasoning_params(
+                params_to_drop,
+                request,
+                _request_model(request),
+                "Azure AI rejected it as an extra parameter its deployment does not pass through.",
+                drop_params_gated=False,
+            )
             for param in params_to_drop:
                 if param in request_data:
                     request_data.pop(param, None)

litellm/llms/base_llm/realtime/transformation.py

@@ -1,10 +1,12 @@
+import dataclasses
 from abc import ABC, abstractmethod
 from collections.abc import Mapping, Sequence
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, Protocol
+from typing import TYPE_CHECKING, Any, Literal, Protocol
 
 import httpx
 
+from litellm._uuid import uuid
 from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
 from litellm.types.realtime import (
     RealtimeInputAudioTranscriptionUsage,
@@ -22,6 +24,26 @@ else:
     LiteLLMLoggingObj = Any
 
 
+@dataclasses.dataclass(frozen=True, slots=True)
+class RealtimeSessionUpdateRejection:
+    code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"]
+    message: str
+    param: str
+
+    def error_event(self, client_event_id: object) -> dict[str, object]:
+        return {
+            "type": "error",
+            "event_id": f"event_{uuid.uuid4()}",
+            "error": {
+                "type": "invalid_request_error",
+                "code": self.code,
+                "message": self.message,
+                "param": self.param,
+                "event_id": client_event_id if isinstance(client_event_id, str) else None,
+            },
+        }
+
+
 class RealtimeBackend(Protocol):
     async def __aenter__(self) -> object: ...
 
@@ -93,6 +115,14 @@ class BaseRealtimeConfig(ABC):
     def session_configuration_request(self, model: str) -> str | None:  # message sent to setup the realtime session
         return None
 
+    def builds_setup_from_session_update(self) -> bool:
+        return False
+
+    def setup_session_update_rejection(
+        self, model: str, session: Mapping[str, object], *, setup_sent: bool
+    ) -> RealtimeSessionUpdateRejection | None:
+        return None
+
     def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
         return None
 

litellm/llms/base_llm/responses/transformation.py

@@ -1,7 +1,7 @@
 import types
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast, get_args
 
 import httpx
 
@@ -27,6 +27,20 @@ else:
     BaseLLMException = Any
 
 ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"]
+ReasoningFieldValues: TypeAlias = Mapping[str, frozenset[str | None] | None]
+REASONING_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
+SUMMARY_ONLY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
+    {"summary": REASONING_SUMMARY_VALUES}
+)
+STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
+    {**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": frozenset({"standard"})}
+)
+STANDARD_MODE_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
+    {**STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, "context": None}
+)
+ANY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
+    {**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": None, "context": None}
+)
 
 
 class BaseResponsesAPIConfig(ABC):
@@ -100,6 +114,18 @@ class BaseResponsesAPIConfig(ABC):
         """The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for."""
         return "detailed"
 
+    def reasoning_field_values(self) -> ReasoningFieldValues | None:
+        return None
+
+    def deployment_names_wrap_openai_models(self) -> bool:
+        return False
+
+    def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
+        return None
+
+    def ignores_unsupported_reasoning_fields(self) -> bool:
+        return False
+
     @abstractmethod
     def map_openai_params(
         self,

litellm/llms/bedrock/chat/chat_completions/transformation.py

@@ -33,6 +33,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
     inline_remote_image_urls,
     inline_remote_media,
 )
+from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
 from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
@@ -43,6 +44,7 @@ from litellm.llms.bedrock.common_utils import (
     bedrock_runtime_chat_completions_serves_reasoning_inline,
     bedrock_serves_service_tier,
     split_bedrock_region_path,
+    strip_bedrock_routing_prefix,
 )
 from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
 from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
@@ -122,6 +124,13 @@ def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) -
     effort: Final = params.get("reasoning_effort")
     if not isinstance(effort, str) or effort not in chat_completions_reasoning_efforts_refused_for(model):
         return params
+    warn_dropped_reasoning(
+        "reasoning_effort",
+        effort,
+        strip_bedrock_routing_prefix(model),
+        "AWS's Chat Completions endpoint rejects that level for this model family, so AWS applies its default effort.",
+        drop_params_gated=False,
+    )
     return _without_params(params, frozenset(("reasoning_effort",)))
 
 
@@ -453,6 +462,14 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
                 ),
                 status_code=400,
             )
+        if malformed_effort:
+            warn_dropped_reasoning(
+                "reasoning_effort",
+                raw_params["reasoning_effort"],
+                strip_bedrock_routing_prefix(model),
+                "Bedrock's Chat Completions endpoint takes it only as a string, so AWS applies its default effort.",
+                drop_params_gated=True,
+            )
         rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]()
         unsupported_tier: Final = _unsupported_service_tier(model, raw_params)
         if unsupported_tier and not (litellm.drop_params or drop_params):

litellm/llms/bedrock/chat/converse_transformation.py

@@ -50,6 +50,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     message_field,
     parts_of,
 )
+from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
 from litellm.llms.anthropic.chat.transformation import (
     DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
     DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
@@ -63,6 +64,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image
 from litellm.llms.bedrock.common_utils import (
     bedrock_chat_uses_native_structured_output,
     bedrock_cleaned_output_schema,
+    bedrock_model_is_unknown,
     bedrock_model_supports_regex_lookaround,
     bedrock_native_output_format,
     bedrock_native_output_is_opt_in,
@@ -187,6 +189,8 @@ def _tools_the_model_accepts(
 
 
 _OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock)
+_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field."
+_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort."
 
 
 def _has_native_text_format(output_config: object) -> bool:
@@ -530,13 +534,14 @@ class AmazonConverseConfig(BaseConfig):
         """
         Handle the reasoning_effort parameter based on the model type.
 
-        - GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields.
+        - GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged
+          via additionalModelRequestFields.
         - OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
         - Nova 2 models: transformed to reasoningConfig.
         - Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
           adaptive Claude 4.6 / 4.7).
         """
-        if "gpt-oss" in model or "deepseek" in model:
+        if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model):
             optional_params["reasoning_effort"] = reasoning_effort
         elif self._is_openai_gpt_reasoning_model(model):
             reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
@@ -639,6 +644,10 @@ class AmazonConverseConfig(BaseConfig):
             or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider)
         )
 
+    def _warn_native_reasoning_drop(self, param: str, value: object, model: str, reason: str) -> None:
+        if param in self.get_supported_openai_params(model):
+            warn_dropped_reasoning(param, value, model, reason, drop_params_gated=False)
+
     def _model_rejects_reasoning_effort_param(self, model: str, base_model: str) -> bool:
         """Whether the model returns a 400 for every ``reasoning_effort`` shape on Converse.
 
@@ -682,7 +691,7 @@ class AmazonConverseConfig(BaseConfig):
             "parallel_tool_calls",
         ]
 
-        if "arn" in model:  # we can't infer the model from the arn, so just add all params
+        if "arn" in model or bedrock_model_is_unknown(model):
             supported_params.append("tools")
             supported_params.append("tool_choice")
             supported_params.append("thinking")
@@ -1142,9 +1151,8 @@ class AmazonConverseConfig(BaseConfig):
                     "tool_choice": {"type": "auto", "disable_parallel_tool_use": not value}
                 }
             if param == "thinking" and drop_thinking_param:
-                verbose_logger.debug(
-                    "Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively.",
-                    model,
+                self._warn_native_reasoning_drop(
+                    param, OPAQUE_VALUE.validate_python(value), model, _DEEPSEEK_THINKING_REASON
                 )
             elif param == "thinking" and not self._is_openai_gpt_reasoning_model(model):
                 if (
@@ -1178,10 +1186,7 @@ class AmazonConverseConfig(BaseConfig):
                         drop_params=drop_params,
                     )
             elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param:
-                verbose_logger.debug(
-                    "Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.",
-                    model,
-                )
+                self._warn_native_reasoning_drop(param, value, model, _DEEPSEEK_R1_EFFORT_REASON)
             elif (
                 param == "reasoning_effort"
                 and isinstance(value, str)

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -2,7 +2,7 @@ import copy
 import json
 import time
 from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, cast, get_args
+from typing import TYPE_CHECKING, Any, Final
 
 import httpx
 from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
@@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
     "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
 )
 
-_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
-    get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
-)
-
 
 def _response_header(headers: Mapping[str, str], name: str) -> str | None:
     return headers.get(name, None)
@@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         """
         return False
 
-    @staticmethod
-    def get_bedrock_invoke_provider(
-        model: str,
-    ) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
-        """
-        Helper function to get the bedrock provider from the model
-
-        handles 4 scenarios:
-        1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
-        2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
-        3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
-        4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
-        """
-        if model.startswith("invoke/"):
-            model = model.replace("invoke/", "", 1)
-
-        # Special case: Check for "nova" in model name first (before "amazon")
-        # This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
-        if "nova" in model.lower():
-            if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
-                return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
-
-        _split_model: Final = model.split(".")[0]
-        if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
-            return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
-
-        # If not a known provider, check for pattern with two slashes
-        provider = AmazonInvokeConfig._get_provider_from_model_path(model)
-        if provider is not None:
-            return provider
-
-        for provider in _INVOKE_PROVIDERS:
-            if provider in model:
-                return provider
-        return None
-
-    @staticmethod
-    def _get_provider_from_model_path(
-        model_path: str,
-    ) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
-        """
-        Helper function to get the provider from a model path with format: provider/model-name
-
-        Args:
-            model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
-
-        Returns:
-            Optional[str]: The provider name, or None if no valid provider found
-        """
-        parts: Final = model_path.split("/")
-        if len(parts) >= 1:
-            provider: Final = parts[0]
-            if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
-                return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
-        return None
-
     def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
         # handle anthropic prompts and amazon titan prompts
         prompt = ""

litellm/llms/bedrock/common_utils.py

@@ -1098,6 +1098,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
     return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
 
 
+def _cost_map_row_names_a_provider(key: str) -> bool:
+    entry: Final = _model_cost_entry(key)
+    return entry is not None and entry.get("litellm_provider") is not None
+
+
 def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
     return tuple(
         _model_cost_entry(key)
@@ -1589,7 +1594,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
 
     The deployment's ``model_info.supports_regex_lookaround`` wins, then the
     ``model_prices_and_context_window.json`` entry of its ``base_model``, then the
-    entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
+    entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
+    no candidate names a model LiteLLM knows, whatever route prefix it carries.
     """
     params: Final = litellm_params or {}
     deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
@@ -1598,7 +1604,41 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
     base_model: Final = params.get("base_model")
     candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model)
     flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates)
-    return next((flag for flag in flags if flag is not None), True)
+    flag: Final = next((flag for flag in flags if flag is not None), None)
+    if flag is not None:
+        return flag
+    return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates)
+
+
+def _bedrock_model_id_without_region(model: str) -> str:
+    routed_id: Final = model.removeprefix("bedrock/")
+    region, separator, regional_id = routed_id.partition("/")
+    return regional_id if separator and region in get_all_bedrock_regions() else routed_id
+
+
+def bedrock_model_is_unknown(model: str) -> bool:
+    model_id: Final = _bedrock_model_id_without_region(model)
+    if "/" in model_id:
+        return False
+    base: Final = get_bedrock_base_model(model_id)
+    regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
+    if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
+        return False
+    from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
+
+    return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None
+
+
+def _names_no_invoke_provider(model: str) -> bool:
+    from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
+
+    model_id: Final = _bedrock_model_id_without_region(model)
+    names_one_model: Final = bool(model_id) and ("/" not in model_id or model_id.startswith("arn:"))
+    return names_one_model and BaseAWSLLM.get_bedrock_invoke_provider(model) is None
+
+
+def bedrock_model_reaches_converse_by_default(model: str) -> bool:
+    return BedrockModelInfo.named_bedrock_route(model, None) is None and _names_no_invoke_provider(model)
 
 
 _BedrockConverseModelFlag: TypeAlias = Literal[
@@ -1607,10 +1647,13 @@ _BedrockConverseModelFlag: TypeAlias = Literal[
 ]
 
 
-def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
+def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]:
     base: Final = get_bedrock_base_model(model)
-    candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))
-    for candidate in candidates:
+    return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))))
+
+
+def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
+    for candidate in _bedrock_model_info_candidates(model):
         with contextlib.suppress(Exception):
             model_info = get_cached_model_info()(
                 model=candidate,
@@ -1788,8 +1831,19 @@ class BedrockModelInfo(BaseLLMModelInfo):
         default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
         ``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
         feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
-        model stays on Converse without the prefix.
+        model stays on Converse without the prefix. A model the cost map does not list as Converse goes to
+        Invoke only when its id names an Invoke provider, and to Converse otherwise.
         """
+        named_route: Final = BedrockModelInfo.named_bedrock_route(model, request_params)
+        if named_route is not None:
+            return named_route
+        return "converse" if _names_no_invoke_provider(model) else "invoke"
+
+    @staticmethod
+    def named_bedrock_route(
+        model: str,
+        request_params: Mapping[str, object] | None,
+    ) -> BedrockRoute | None:
         route_mappings: dict[
             str,
             Literal[
@@ -1844,7 +1898,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
             return "converse"
         if _OPENAI_FAMILY_MODEL_RE.search(base_model):
             return "converse"
-        return "invoke"
+        return None
 
     @staticmethod
     def _explicit_converse_route(model: str) -> bool:

litellm/llms/bedrock/realtime/handler.py

@@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm._logging import redact_string, verbose_proxy_logger
-from litellm._uuid import uuid
 from litellm.constants import (
     BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
     BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
@@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
+from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection
 from litellm.types.llms.bedrock import AwsAuthParams
 from litellm.types.llms.openai import OpenAIRealtimeEvents
 from litellm.types.realtime import RealtimeResponseTransformInput
 
 from ..base_aws_llm import BaseAWSLLM, run_aws_signing
 from ..common_utils import BedrockError
-from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
+from .transformation import BedrockRealtimeConfig, requested_modalities
 
 _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
 
@@ -256,23 +256,11 @@ async def _ack_session_update(
 async def _reject_session_update(
     client_ws: RealtimeClientWebSocket,
     parsed_client_message: Mapping[str, JsonValue],
-    rejection: BedrockSessionUpdateRejection,
+    rejection: RealtimeSessionUpdateRejection,
 ) -> bool:
     """Send an error event for a session.update Bedrock cannot apply; False means the client is gone."""
-    client_event_id: Final = parsed_client_message.get("event_id")
-    error_event: Final = {
-        "type": "error",
-        "event_id": f"event_{uuid.uuid4()}",
-        "error": {
-            "type": "invalid_request_error",
-            "code": rejection.code,
-            "message": rejection.message,
-            "param": rejection.param,
-            "event_id": client_event_id if isinstance(client_event_id, str) else None,
-        },
-    }
     try:
-        await client_ws.send_text(json.dumps(error_event))
+        await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id"))))
     except Exception as e:  # noqa: BLE001  # any send failure means the client is gone
         verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
         return False

litellm/llms/bedrock/realtime/transformation.py

@@ -11,7 +11,7 @@ import math
 import uuid as uuid_lib
 from collections.abc import Mapping, Sequence
 from types import MappingProxyType
-from typing import Final, Literal
+from typing import Final
 
 import httpx
 from pydantic import ConfigDict, JsonValue, TypeAdapter
@@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
+from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
 from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
 from litellm.types.llms.base import LiteLLMBaseModel
@@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[
     return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
 
 
-@dataclasses.dataclass(frozen=True, slots=True)
-class BedrockSessionUpdateRejection:
-    code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
-    message: str
-    param: str
-
-
 @dataclasses.dataclass(frozen=True, slots=True)
 class BedrockLockedSessionSettings:
     """The session settings Bedrock Nova Sonic locks in once the prompt starts."""
@@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             max_tokens=max_tokens,
         )
 
-    def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None:
+    def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None:
         """The error for a session.update this session cannot apply, or None when it applies."""
         session_config: Final = json_message.get("session")
         if not isinstance(session_config, Mapping):
@@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         malformed: Final = () if self.drop_params else _malformed_settings(session_config)
         if malformed:
             name, expected = malformed[0]
-            return BedrockSessionUpdateRejection(
+            return RealtimeSessionUpdateRejection(
                 code="invalid_type",
                 message=(
                     f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
@@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             )
         unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
         if unplayable_voice is not None:
-            return BedrockSessionUpdateRejection(
+            return RealtimeSessionUpdateRejection(
                 code="invalid_value",
                 message=(
                     f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
@@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         if not self.prompt_started:
             return None
         if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings:
-            return BedrockSessionUpdateRejection(
+            return RealtimeSessionUpdateRejection(
                 code="session_update_not_applied",
                 message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied",
                 param="session",

litellm/llms/chatgpt/authenticator.py

@@ -122,6 +122,12 @@ class Authenticator:
         return tokens["access_token"]
 
     def get_account_id(self, access_token: str | None = None) -> str | None:
+        return self._account_id(access_token, persist_derived=True)
+
+    def read_account_id(self, access_token: str | None = None) -> str | None:
+        return self._account_id(access_token, persist_derived=False)
+
+    def _account_id(self, access_token: str | None, persist_derived: bool) -> str | None:
         auth_data: Final = self._read_auth_file()
         if access_token and (not auth_data or auth_data.get("access_token") != access_token):
             return extract_chatgpt_account_id(access_token)
@@ -133,7 +139,7 @@ class Authenticator:
         id_token: Final = auth_data.get("id_token")
         saved_access_token: Final = auth_data.get("access_token")
         derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token))
-        if derived:
+        if derived and persist_derived:
             self._write_auth_file({**auth_data, "account_id": derived})
         return derived
 

litellm/llms/chatgpt/responses/transformation.py

@@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
     safe_convert_created_field,
 )
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.custom_httpx.sse_lines import split_sse_lines
 from litellm.llms.openai.common_utils import OpenAIError
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
@@ -295,3 +296,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def supports_native_websocket(self) -> bool:
         """ChatGPT does not support native WebSocket for Responses API"""
         return False
+
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return STANDARD_MODE_REASONING_FIELD_VALUES

litellm/llms/custom_httpx/llm_http_handler.py

@@ -249,7 +249,7 @@ if TYPE_CHECKING:
         FakeAnthropicMessagesStreamIterator,
     )
     from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
-    from litellm.responses.streaming_iterator import ResponsesWsFrameGuard
+    from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission
     from litellm.types.google_genai.main import GenerateContentResponse
     from litellm.types.llms.openai_evals import (
         CancelEvalResponse,
@@ -6973,14 +6973,19 @@ class BaseLLMHTTPHandler:
                     ),
                 )
 
-                # Auto-send session setup if the provider requires it (e.g.
-                # Gemini/Vertex AI Live needs a `setup` before any realtime_input).
+                # Auto-send session setup at connect unless the provider builds it from the
+                # client's first session.update (Gemini/Vertex AI Live accept exactly one
+                # setup, so RealTimeStreaming sends theirs on the first client message).
                 # Build the streaming handler first so a transcription guardrail's
                 # auto-response disable can be folded into this one setup: Gemini
                 # rejects a second setup, so a follow-up disable would be dropped
                 # and the guardrail bypassed.
                 _session_config: str | None = None
-                if provider_config.requires_session_configuration():
+                sends_setup_at_connect: Final = (
+                    provider_config.requires_session_configuration()
+                    and not provider_config.builds_setup_from_session_update()
+                )
+                if sends_setup_at_connect:
                     _session_config = provider_config.session_configuration_request(model)
                     if _session_config:
                         _session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable(
@@ -7264,7 +7269,7 @@ class BaseLLMHTTPHandler:
         custom_llm_provider: str | None = None,
         first_message: str | None = None,
         request_defaults: ResponsesWebSocketRequestDefaults | None = None,
-        turn_admission: Callable[[], Awaitable[None]] | None = None,
+        turn_admission: "ResponsesWsTurnAdmission | None" = None,
         frame_guard: "ResponsesWsFrameGuard | None" = None,
         **kwargs: object,
     ) -> Exception | None:

litellm/llms/deepseek/chat/transformation.py

@@ -6,10 +6,12 @@ from collections.abc import Coroutine, Mapping, Sequence
 from typing import Final, Literal, cast, overload
 
 import litellm
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     convert_content_list_to_str,
     extract_search_results_text,
 )
+from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
@@ -18,6 +20,16 @@ from litellm.utils import supports_reasoning, supports_vision
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
 SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra")
+_NO_BUDGET_REASON: Final = "DeepSeek takes thinking only as enabled or disabled, with no budget."
+
+
+def _warn_on_dropped_budget(thinking: object, model: str) -> None:
+    fields: Final = as_str_mapping(thinking)
+    if fields is None or fields.get("type") != "enabled" or "budget_tokens" not in fields:
+        return
+    warn_dropped_reasoning(
+        "thinking.budget_tokens", fields["budget_tokens"], model, _NO_BUDGET_REASON, drop_params_gated=False
+    )
 
 
 class DeepSeekChatConfig(OpenAIGPTConfig):
@@ -53,6 +65,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
         # Then re-add only if valid for DeepSeek
         thinking_value: Final = optional_params.pop("thinking", None)
         reasoning_effort: Final = optional_params.pop("reasoning_effort", None)
+        _warn_on_dropped_budget(OPAQUE_VALUE.validate_python(thinking_value), model)
 
         # Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens
         if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"):

litellm/llms/fal_ai/chat/transformation.py

@@ -12,7 +12,9 @@ from typing import TYPE_CHECKING, Final
 import httpx
 from pydantic import ConfigDict, TypeAdapter
 
+import litellm
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.base import LiteLLMBaseModel
@@ -29,6 +31,10 @@ PROVIDER_PREFIX: Final[str] = "fal_ai/"
 PASSTHROUGH_PARAMS: Final[frozenset[str]] = frozenset(("reasoning", "temperature", "top_p"))
 REASONING_DISABLED_EFFORTS: Final[frozenset[str]] = frozenset(("none", "minimal"))
 REASONING_ENABLED_EFFORTS: Final[frozenset[str]] = frozenset(("low", "medium", "high"))
+_UNSUPPORTED_EFFORT_REASON: Final = (
+    f"fal only turns reasoning off for {sorted(REASONING_DISABLED_EFFORTS)} "
+    f"and on for {sorted(REASONING_ENABLED_EFFORTS)}."
+)
 
 
 class _FalUsage(LiteLLMBaseModel):
@@ -120,9 +126,10 @@ class FalAIChatConfig(BaseConfig):
             return False
         if isinstance(value, str) and value in REASONING_ENABLED_EFFORTS:
             return True
-        if drop_params:
-            return None
-        raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}")
+        if not (drop_params or litellm.drop_params is True):
+            raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}")
+        warn_dropped_reasoning("reasoning_effort", value, model, _UNSUPPORTED_EFFORT_REASON, drop_params_gated=True)
+        return None
 
     def _translate_param(self, param: str, value: object, model: str, drop_params: bool) -> tuple[str, object] | None:
         if param in ("temperature", "top_p"):

litellm/llms/fireworks_ai/chat/transformation.py

@@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     strip_litellm_response_only_fields,
     unpack_legacy_defs,
 )
+from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import (
@@ -97,6 +98,7 @@ def _json_schema_response_format(schema: object, name: str) -> Mapping[str, obje
 
 
 EFFORT_KWARG_KEYS: Final = frozenset({"enable_thinking", "thinking", "reasoning_budget", "low_effort"})
+_NOT_A_REASONING_MODEL_REASON: Final = "litellm's model map does not list it as a reasoning model."
 
 
 def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bool | None:
@@ -107,6 +109,13 @@ def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bo
     return None
 
 
+def warn_dropped_effort_kwargs(chat_template_kwargs: object, effort: object, model: str) -> None:
+    if effort != "none":
+        warn_dropped_reasoning(
+            "chat_template_kwargs", chat_template_kwargs, model, _NOT_A_REASONING_MODEL_REASON, drop_params_gated=False
+        )
+
+
 def effort_from_chat_template_kwargs(kwargs: Mapping[str, object]) -> object:
     enable_thinking: Final = _bool_from_kwargs(kwargs, ("enable_thinking", "thinking"))
     if enable_thinking is False:
@@ -438,10 +447,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
         if effort is None:
             return ()
         if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
-            verbose_logger.debug(
-                "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.",
-                model,
-            )
+            warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model)
             return ()
         return (("reasoning_effort", effort),)
 

litellm/llms/fireworks_ai/completion/transformation.py

@@ -2,6 +2,7 @@ from collections.abc import Mapping
 from typing import Final
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
 from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage
 from litellm.utils import supports_reasoning
 
@@ -12,6 +13,7 @@ from ..chat.transformation import (
     NIM_VLLM_STRIP_PARAMS,
     FireworksAIConfig,
     effort_from_chat_template_kwargs,
+    warn_dropped_effort_kwargs,
 )
 from ..common_utils import FireworksAIMixin, resolve_fireworks_resource_name
 
@@ -129,10 +131,7 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig
             )
             return result
         if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"):
-            verbose_logger.debug(
-                "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.",
-                model,
-            )
+            warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model)
             return result
         return {**result, "reasoning_effort": effort}
 

litellm/llms/fireworks_ai/responses/transformation.py

@@ -6,6 +6,10 @@ from urllib.parse import unquote
 import httpx
 
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.base_llm.responses.transformation import (
+    STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
+    ReasoningFieldValues,
+)
 from litellm.llms.fireworks_ai.common_utils import (
     FIREWORKS_FORWARD_USER_ID_PARAM,
     get_fireworks_forwarded_user_id,
@@ -129,3 +133,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
     def supports_native_file_search(self) -> bool:
         return False
+
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
+
+    def ignores_unsupported_reasoning_fields(self) -> bool:
+        return True

litellm/llms/gemini/realtime/transformation.py

@@ -7,6 +7,7 @@ from collections import OrderedDict
 from collections.abc import Mapping, Sequence
 from typing import Final, cast
 
+from pydantic import TypeAdapter
 from typing_extensions import ReadOnly, Required, TypedDict
 
 import litellm
@@ -15,8 +16,9 @@ from litellm._uuid import uuid
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import join_api_base
-from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
+from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
 from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
+    OPENAI_TO_GEMINI_TTS_VOICES,
     VertexGeminiConfig,
 )
 from litellm.responses.litellm_completion_transformation.transformation import (
@@ -76,27 +78,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
 
 # Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped.
 _KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
+_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"})
 
 
 OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
     {"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
 )
 
+# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28)
+GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset(
+    {
+        "achernar",
+        "achird",
+        "algenib",
+        "algieba",
+        "alnilam",
+        "aoede",
+        "autonoe",
+        "callirrhoe",
+        "charon",
+        "despina",
+        "enceladus",
+        "erinome",
+        "fenrir",
+        "gacrux",
+        "iapetus",
+        "kore",
+        "laomedeia",
+        "leda",
+        "orus",
+        "puck",
+        "pulcherrima",
+        "rasalgethi",
+        "sadachbia",
+        "sadaltager",
+        "schedar",
+        "sulafat",
+        "umbriel",
+        "vindemiatrix",
+        "zephyr",
+        "zubenelgenubi",
+    }
+)
 
-def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
+
+def _is_undocumented_live_voice(voice: object) -> bool:
+    return (
+        isinstance(voice, str)
+        and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
+        and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
+    )
+
+
+def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None:
     """Build the Gemini Live speechConfig for a client-requested voice.
 
-    OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
-    the session on an unknown voice, so they are dropped with a warning and
-    the model keeps its default voice. Every other name is forwarded verbatim.
+    OpenAI stock voice names have no Gemini equivalent, so they are always
+    dropped, and so is any name outside Google's documented Live voices when
+    ``drop_undocumented`` is set, keeping the model's default voice. Every
+    other name is forwarded verbatim.
     """
-    if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
+    if not isinstance(voice, str):
+        return VertexGeminiConfig().map_audio_params({"voice": voice})
+    if voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
         verbose_logger.warning(
             "Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
             "dropping it so the session keeps the model's default voice.",
             voice,
         )
         return None
+    if drop_undocumented and _is_undocumented_live_voice(voice):
+        verbose_logger.warning(
+            "Gemini Realtime: voice %s is not a documented Gemini Live voice; "
+            "drop_params is set, so it is dropped and the session keeps the model's default voice.",
+            voice,
+        )
+        return None
     return VertexGeminiConfig().map_audio_params({"voice": voice})
 
 
@@ -142,16 +199,34 @@ def _base64_decoded_byte_count(data: str) -> int:
     return max(len(data) * 3 // 4 - padding, 0)
 
 
+_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
+
+
+def _chat_completion_tool(tool: object) -> dict[str, object]:
+    fields: Final = as_str_mapping(tool) or {}
+    if fields.get("type") != "function" or "function" in fields:
+        return {**fields}
+    return {"type": "function", "function": {key: value for key, value in fields.items() if key != "type"}}
+
+
+def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None:
+    parts: Final = system_instruction.get("parts", []) if system_instruction else []
+    texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str))
+    return "\n".join(texts) if texts else None
+
+
 class GeminiRealtimeConfig(BaseRealtimeConfig):
     _TOOL_CALL_ID_TO_NAME_MAX = 256  # LRU cap for call_id→name mapping
 
-    def __init__(self):
+    def __init__(self, drop_params: bool = False):
         super().__init__()
+        self.drop_params = drop_params
         self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict()
         # Gemini Live sometimes emits usageMetadata in a standalone frame between
         # turns; buffer it here so the next response.done carries the token counts.
         self._pending_usage_metadata: dict | None = None
         self._unbilled_input_audio_bytes: int = 0
+        self._manual_activity_open: bool = False
 
     def is_setup_message(self, msg_obj: dict) -> bool:
         return "setup" in msg_obj
@@ -167,6 +242,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         """Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
         return True
 
+    def _rejects_undocumented_voices(self, model: str) -> bool:
+        """Google AI Studio closes the socket (1007) on an unknown voice only for its native-audio Live models."""
+        return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
+
     @staticmethod
     def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]:
         detail_mapping: Final = as_str_mapping(details)
@@ -272,17 +351,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         except (json.JSONDecodeError, TypeError, AttributeError):
             return False
 
+    def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]:
+        if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request):
+            return ()
+        self._manual_activity_open = True
+        activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}}
+        return (json.dumps({"realtimeInput": activity_start}),)
+
+    def _end_manual_activity(self) -> tuple[str, ...]:
+        if not self._manual_activity_open:
+            return ()
+        self._manual_activity_open = False
+        activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}}
+        return (json.dumps({"realtimeInput": activity_end}),)
+
     def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
         """Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
         if self._manual_turn_detection_enabled(session_configuration_request):
-            realtime_input_dict: BidiGenerateContentRealtimeInput = {
-                "activityEnd": True,
-            }
-            verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
-        else:
-            realtime_input_dict = {"audioStreamEnd": True}
-            verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend")
-        return [json.dumps({"realtimeInput": realtime_input_dict})]
+            return list(self._end_manual_activity())
+        audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True}
+        return [json.dumps({"realtimeInput": audio_stream_end})]
 
     def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection:
         """Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``.
@@ -325,7 +413,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
             "voice",
         ]
 
-    def map_openai_params(self, optional_params: dict, non_default_params: dict) -> dict:
+    def map_openai_params(self, optional_params: dict, non_default_params: dict, *, model: str) -> dict:
         if "generationConfig" not in optional_params:
             optional_params["generationConfig"] = {}
         for key, value in non_default_params.items():
@@ -333,12 +421,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                 optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}])
             elif key == "temperature":
                 optional_params["generationConfig"]["temperature"] = value
-            elif key == "max_response_output_tokens":
+            elif key == "max_response_output_tokens" and isinstance(value, int):
                 optional_params["generationConfig"]["maxOutputTokens"] = value
             elif key == "modalities":
-                optional_params["generationConfig"]["responseModalities"] = [
-                    modality.upper() for modality in cast(list[str], value)
-                ]
+                optional_params["generationConfig"]["responseModalities"] = list(
+                    GeminiRealtimeConfig._requested_response_modalities(cast(object, value))
+                )
             elif key == "tools":
                 from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
                     VertexGeminiConfig,
@@ -347,34 +435,43 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                 vertex_gemini_config = VertexGeminiConfig()
                 # Tools should be at the top level of setup, not inside generationConfig
                 optional_params["tools"] = vertex_gemini_config.map_function(
-                    value=value, optional_params=optional_params
+                    value=[_chat_completion_tool(tool) for tool in _JSON_ARRAY.validate_python(value)],
+                    optional_params=optional_params,
                 )
             elif key == "input_audio_transcription" and value is not None:
                 optional_params["inputAudioTranscription"] = {}
-            elif key == "turn_detection" and value is not None:
-                value_typed = cast(OpenAIRealtimeTurnDetection, value)
-                if (
-                    isinstance(value_typed, dict)
-                    and value_typed.get("type") == "semantic_vad"
-                    and "create_response" not in value_typed
-                ):
-                    # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
-                    # Only skip when there is no create_response override so that
-                    # a guardrail-injected create_response:false is not dropped.
-                    continue
-                transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed)
-                if transformed_audio_activity_config:
-                    optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig(
-                        automaticActivityDetection=transformed_audio_activity_config
-                    )
+            elif key == "turn_detection":
+                realtime_input_config = self._realtime_input_config(cast(object, value))
+                if realtime_input_config is not None:
+                    optional_params["realtimeInputConfig"] = realtime_input_config
             elif key == "voice":
-                speech_config = _gemini_live_speech_config(value)
+                speech_config = _gemini_live_speech_config(
+                    value, self.drop_params and self._rejects_undocumented_voices(model)
+                )
                 if speech_config:
                     optional_params["generationConfig"]["speechConfig"] = speech_config
         if len(optional_params["generationConfig"]) == 0:
             optional_params.pop("generationConfig")
         return optional_params
 
+    def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None:
+        if turn_detection is None:
+            return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True})
+        value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection)
+        if (
+            isinstance(value_typed, dict)
+            and value_typed.get("type") == "semantic_vad"
+            and "create_response" not in value_typed
+        ):
+            # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
+            # Only skip when there is no create_response override so that
+            # a guardrail-injected create_response:false is not dropped.
+            return None
+        transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed)
+        if not transformed_audio_activity_config:
+            return None
+        return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config)
+
     @staticmethod
     def _extract_turn_detection(session: dict) -> dict | None:
         """Extract turn_detection from a session.update payload.
@@ -423,6 +520,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
             if isinstance(input_cfg, dict):
                 if "input_audio_transcription" not in normalized and "transcription" in input_cfg:
                     normalized["input_audio_transcription"] = input_cfg["transcription"]
+                if "turn_detection" not in normalized and "turn_detection" in input_cfg:
+                    normalized["turn_detection"] = input_cfg["turn_detection"]
             output_cfg: Final = audio.get("output")
             if isinstance(output_cfg, dict) and output_cfg.get("voice"):
                 normalized["voice"] = output_cfg["voice"]
@@ -434,7 +533,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         return normalized
 
     @staticmethod
-    def _model_cost_entry(model: str) -> dict:
+    def _model_cost_entry(model: str) -> Mapping[str, object]:
         entry = litellm.model_cost.get(model)
         if entry is None:
             stripped: Final = model.split("/", 1)[-1]
@@ -455,18 +554,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
 
     @staticmethod
-    def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
-        """Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
-        audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
-        normalized: Final = tuple(
-            modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
+    def _requested_response_modalities(value: object) -> tuple[str, ...]:
+        if isinstance(value, str):
+            return (value.upper(),)
+        if not isinstance(value, (list, tuple)):
+            return ()
+        return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value))
+
+    @staticmethod
+    def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]:
+        unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES)
+        if unknown:
+            return unknown
+        if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)):
+            return ("TEXT",)
+        if GeminiRealtimeConfig._is_text_only_live_model(model) and frozenset(requested) == frozenset(("AUDIO",)):
+            return ("AUDIO",)
+        return ()
+
+    def setup_session_update_rejection(
+        self, model: str, session: Mapping[str, object], *, setup_sent: bool
+    ) -> RealtimeSessionUpdateRejection | None:
+        if self.drop_params:
+            return None
+        if setup_sent:
+            return RealtimeSessionUpdateRejection(
+                code="session_update_not_applied",
+                message=(
+                    "Gemini Live accepts one session configuration per connection, and this session already sent "
+                    "its configuration, so this session.update was not applied. Put every setting in session.update "
+                    "before the first conversation content, or set drop_params to drop it instead"
+                ),
+                param="session",
+            )
+        return self._modality_rejection(model, session) or self._voice_rejection(model, session)
+
+    @staticmethod
+    def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
+        param: Final = "output_modalities" if "output_modalities" in session else "modalities"
+        unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities(
+            model, GeminiRealtimeConfig._requested_response_modalities(session.get(param))
         )
+        if not unsupported:
+            return None
+        return RealtimeSessionUpdateRejection(
+            code="invalid_value",
+            message=(
+                f"Invalid value for 'session.{param}': {model} cannot produce "
+                f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead"
+            ),
+            param=f"session.{param}",
+        )
+
+    def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
+        audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {}
+        output_voice: Final = audio_output.get("voice")
+        voice: Final = output_voice or session.get("voice")
+        if not (self._rejects_undocumented_voices(model) and _is_undocumented_live_voice(voice)):
+            return None
+        param: Final = "session.audio.output.voice" if output_voice else "session.voice"
+        return RealtimeSessionUpdateRejection(
+            code="invalid_value",
+            message=(
+                f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. "
+                "Set drop_params to drop it instead"
+            ),
+            param=param,
+        )
+
+    @staticmethod
+    def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
+        """Drop modalities Gemini Live cannot produce at all, swap the ones this
+        model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for
+        text-only ones such as transcribe-live), then keep a single modality
+        because Gemini Live rejects a setup naming two."""
+        return GeminiRealtimeConfig._single_response_modality(
+            GeminiRealtimeConfig._swap_unsupported_response_modalities(
+                model, GeminiRealtimeConfig._known_response_modalities(model, modalities)
+            )
+        )
+
+    @staticmethod
+    def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
+        normalized: Final = tuple(str(modality).upper() for modality in modalities)
+        known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES)
+        if len(known) < len(normalized):
+            verbose_logger.warning(
+                "Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s",
+                model,
+                ", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES),
+            )
+        return known or (GeminiRealtimeConfig._default_response_modality(model),)
+
+    @staticmethod
+    def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]:
         if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
+            verbose_logger.warning(
+                "Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO",
+                model,
+            )
             return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
         if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
             return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
         return normalized
 
+    @staticmethod
+    def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]:
+        if len(modalities) <= 1:
+            return modalities
+        chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0]
+        verbose_logger.warning(
+            "Gemini Live: at most one response modality is allowed per session; using %s for the requested %s",
+            chosen,
+            ", ".join(modalities),
+        )
+        return (chosen,)
+
     @staticmethod
     def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]:
         generation_config: Final = setup.get("generationConfig")
@@ -504,7 +707,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         # would be silently dropped because ``map_openai_params`` only
         # recognises the flat OpenAI-beta key names.
         session_payload = self._normalize_session_payload_for_mapping(session_payload)
-        new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload)
+        new_overrides: Final = self.map_openai_params(
+            optional_params={}, non_default_params=session_payload, model=model
+        )
 
         if session_configuration_request is None:
             generation_config: Final = new_overrides.setdefault("generationConfig", {})
@@ -618,14 +823,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                 message_str = str(message)
             raise ValueError(f"Invalid JSON message: {message_str}")
 
-        messages: Final[list[str]] = []
         msg_type: Final = json_message.get("type")
 
         if msg_type == "session.update":
             return self._handle_session_update(json_message, model, session_configuration_request)
 
         if msg_type == "response.create":
-            return []  # Gemini responds automatically; nothing to forward
+            return list(self._end_manual_activity())
 
         if msg_type == "conversation.item.create":
             return self._handle_conversation_item(json_message)
@@ -643,8 +847,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
 
             gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict})
             verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend")
-            messages.append(gemini_msg)
-            return messages
+            return [*self._start_manual_activity(session_configuration_request), gemini_msg]
 
         if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"):
             return self._handle_input_audio_buffer_commit_or_end(session_configuration_request)
@@ -660,22 +863,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         logging_session_id: str,
         session_configuration_request: str | None = None,
     ) -> OpenAIRealtimeStreamSessionEvents:
-        if session_configuration_request:
-            session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
-        else:
-            session_configuration_request_dict = {}
+        session_configuration_request_dict: Final = _parse_setup(
+            session_configuration_request or self.session_configuration_request(model)
+        )
 
         _model: Final = session_configuration_request_dict.get("model") or model
         generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {}
         gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"])
         _modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)]
-        _system_instruction: Final = session_configuration_request_dict.get("systemInstruction")
+        instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction"))
         session: Final = OpenAIRealtimeStreamSession(
             id=logging_session_id,
             modalities=_modalities,
         )
-        if _system_instruction is not None and isinstance(_system_instruction, str):
-            session["instructions"] = _system_instruction
+        if instructions is not None:
+            session["instructions"] = instructions
         if _model is not None and isinstance(_model, str):
             # Strip Vertex/AI Studio path prefixes to expose the bare model name.
             if "/models/" in _model:
@@ -1662,6 +1864,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         # Deferred setup opt-in: litellm.gemini_live_defer_setup = True
         return not litellm.gemini_live_defer_setup
 
+    def builds_setup_from_session_update(self) -> bool:
+        return True
+
     def session_configuration_request(self, model: str) -> str:
         """
 

litellm/llms/github_copilot/responses/transformation.py

@@ -24,6 +24,7 @@ from litellm._logging import verbose_logger
 from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
 from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.types.llms.openai import (
     ResponseInputParam,
@@ -437,6 +438,12 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
         return False
 
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return STANDARD_MODE_REASONING_FIELD_VALUES
+
+    def ignores_unsupported_reasoning_fields(self) -> bool:
+        return True
+
     def supports_native_websocket(self) -> bool:
         """GitHub Copilot does not support native WebSocket for Responses API"""
         return False

litellm/llms/hosted_vllm/chat/transformation.py

@@ -99,7 +99,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
         thinking_effort: Final = (
             None
             if "reasoning_effort" in non_default_params
-            else reasoning_effort_from_thinking(thinking, drop_params=drop_params)
+            else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params)
         )
         if thinking_effort is not None:
             non_default_params["reasoning_effort"] = thinking_effort

litellm/llms/litellm_proxy/responses/transformation.py

@@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError
 
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.responses.transformation import ANY_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.utils import LlmProviders
@@ -42,6 +43,9 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def custom_llm_provider(self) -> LlmProviders:
         return LlmProviders.LITELLM_PROXY
 
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return ANY_REASONING_FIELD_VALUES
+
     def get_complete_url(
         self,
         api_base: str | None,

litellm/llms/mistral/chat/transformation.py

@@ -9,17 +9,23 @@ Docs - https://docs.mistral.ai/api/
 from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
 from itertools import accumulate, count
 from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
+from urllib.parse import urlparse
 
 import httpx
+from pydantic import TypeAdapter
 from typing_extensions import TypeIs  # noqa: TID251  # TypeIs reaches typing only on 3.13
 
+import litellm
 from litellm._logging import verbose_logger
+from litellm.exceptions import UnsupportedParamsError
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     convert_content_list_to_str,
     strip_litellm_internal_message_fields,
     strip_none_values_from_message,
 )
+from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.llms.openai.chat.gpt_transformation import (
     OpenAIChatCompletionStreamingHandler,
@@ -30,7 +36,15 @@ from litellm.router_utils.reasoning_effort_capability import (
     nearest_declared_reasoning_effort,
 )
 from litellm.secret_managers.main import get_secret_str
-from litellm.types.llms.mistral import MistralTextBlock, MistralThinkingBlock, MistralToolCallMessage
+from litellm.types.llms.mistral import (
+    MistralCacheKeyParams,
+    MistralFilePart,
+    MistralReasoningEffortParams,
+    MistralRequestRouting,
+    MistralTextBlock,
+    MistralThinkingBlock,
+    MistralToolCallMessage,
+)
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject
 from litellm.types.utils import ModelResponse, ModelResponseStream
 from litellm.utils import convert_to_model_response_object, supports_reasoning
@@ -57,6 +71,33 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
 
 
 _NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
+_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting)
+_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams)
+_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams)
+_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart)
+_REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system prompt instead of the requested level."
+_NON_LEVEL_EFFORT_REASON: Final = (
+    "the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort."
+)
+_DOCUMENTED_MAGISTRAL_EFFORTS: Final = frozenset({"low", "medium", "high"})
+_REASONING_ON_THINKING_TYPES: Final = frozenset({"enabled", "adaptive"})
+
+
+def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]:
+    requested: Final = as_str_mapping(reasoning_effort)
+    if requested is None:
+        return reasoning_effort, None
+    return requested.get("effort"), requested.get("summary")
+
+
+def _warn_unless_reasoning_prompt_honors(param: str, value: object, model: str) -> None:
+    if param == "reasoning_effort" and isinstance(value, str) and value in _DOCUMENTED_MAGISTRAL_EFFORTS:
+        return
+    thinking: Final = as_str_mapping(value)
+    thinking_type: Final = thinking.get("type") if thinking is not None else None
+    if param == "thinking" and isinstance(thinking_type, str) and thinking_type in _REASONING_ON_THINKING_TYPES:
+        return
+    warn_dropped_reasoning(param, value, model, _REASONING_PROMPT_REASON, drop_params_gated=False)
 
 
 def _is_json_array(value: object) -> TypeIs[Sequence[object]]:  # guard-ok: a decoded JSON array is a list
@@ -88,6 +129,8 @@ class MistralConfig(OpenAIGPTConfig):
     - `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
     """
 
+    _is_base_class = False
+
     temperature: int | None = None
     top_p: int | None = None
     max_tokens: int | None = None
@@ -131,6 +174,7 @@ class MistralConfig(OpenAIGPTConfig):
     def get_supported_openai_params(self, model: str) -> list[str]:
         is_magistral: Final = "magistral" in model.lower()
         accepts_reasoning_effort: Final = self._accepts_reasoning(model)
+        accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral"
         return [
             "stream",
             "temperature",
@@ -145,6 +189,7 @@ class MistralConfig(OpenAIGPTConfig):
             "parallel_tool_calls",
             *(("thinking",) if is_magistral else ()),
             *(("reasoning_effort",) if accepts_reasoning_effort else ()),
+            *(("prompt_cache_key",) if accepts_prompt_cache_key else ()),
         ]
 
     def _map_tool_choice(self, tool_choice: str) -> str:
@@ -211,15 +256,90 @@ class MistralConfig(OpenAIGPTConfig):
                 optional_params["extra_body"] = {"random_seed": value}
             if param == "response_format":
                 optional_params["response_format"] = value
-            if param == "reasoning_effort" and "magistral" not in model.lower():
-                optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider)
+            if (
+                param == "reasoning_effort"
+                and "magistral" not in model.lower()
+                and (
+                    reasoning_level := self._reasoning_level_to_send(
+                        model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params
+                    )
+                )
+                is not None
+            ):
+                optional_params["reasoning_effort"] = reasoning_level
             if param in ("reasoning_effort", "thinking") and "magistral" in model.lower():
                 # Flag that we need to add reasoning system prompt
                 optional_params["_add_reasoning_prompt"] = True
-            if param == "parallel_tool_calls":
-                optional_params["parallel_tool_calls"] = value
+                _warn_unless_reasoning_prompt_honors(param, OPAQUE_VALUE.validate_python(value), model)
+            if param in ("parallel_tool_calls", "prompt_cache_key"):
+                optional_params[param] = value
         return optional_params
 
+    def _reasoning_level_to_send(
+        self, model: str, requested: MistralReasoningEffortParams, drop_params: bool
+    ) -> str | None:
+        requested_effort: Final = requested.get("reasoning_effort")
+        level, summary = _reasoning_level_and_summary(requested_effort)
+        sendable: Final = (
+            _accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None
+        )
+        if sendable is not None and summary is None:
+            return sendable
+        if drop_params or litellm.drop_params is True:
+            if sendable is None and level is not None:
+                warn_dropped_reasoning(
+                    "reasoning_effort", requested_effort, model, _NON_LEVEL_EFFORT_REASON, drop_params_gated=True
+                )
+            return sendable
+        raise UnsupportedParamsError(
+            message=(
+                f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, "
+                "since the Mistral API takes it only as a level such as 'high', with no reasoning summary. "
+                "To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
+            ),
+            llm_provider=self.custom_llm_provider,
+            model=model,
+        )
+
+    def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool:
+        resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None)
+        hostname: Final = urlparse(resolved_api_base).hostname or ""
+        return hostname == "mistral.ai" or hostname.endswith(".mistral.ai")
+
+    @staticmethod
+    def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool:
+        return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii()
+
+    def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None:
+        if not self.accepts_prompt_cache_key_at(api_base):
+            return "since only the Mistral API (*.mistral.ai) takes it"
+        if not self.takes_prompt_cache_key_value(prompt_cache_key):
+            return "since the Mistral API takes it only as a string of ASCII characters"
+        return None
+
+    def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool:
+        requested_api_base: Final = routing.get("api_base")
+        api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None
+        allowed_openai_params: Final = routing.get("allowed_openai_params")
+        if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params:
+            return False
+        rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base)
+        if rejection is None:
+            return False
+        if litellm.drop_params is True or routing.get("drop_params") is True:
+            return True
+        requested_provider: Final = routing.get("custom_llm_provider")
+        provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider
+        raise UnsupportedParamsError(
+            message=(
+                f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. "
+                "To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set "
+                "`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
+            ),
+            llm_provider=provider,
+            model=model,
+        )
+
     def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
         # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai
         api_base = (
@@ -359,24 +479,31 @@ class MistralConfig(OpenAIGPTConfig):
         messages = self._handle_message_with_file(messages)
         return messages
 
+    @staticmethod
+    def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]:
+        file_part: Final = block.get("file") if block.get("type") == "file" else None
+        file_id: Final = (
+            _MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None
+        )
+        if not file_id:
+            return block
+        return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id}
+
+    @classmethod
+    def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues:
+        content: Final = message.get("content")
+        if not isinstance(content, list):
+            return message
+        blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content)
+        return cast(  # cast-ok: only file blocks change, to the file_id shape the Mistral API takes
+            AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]}
+        )
+
     def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]:
         """
         Mistral API supports only 'file_id' in message content with type 'file'.
         """
-        for m in messages:
-            _content_block = m.get("content")
-            if _content_block and isinstance(_content_block, list):
-                if any(c.get("type") == "file" for c in _content_block):
-                    # If file content is present, we get file_id from 'file' attribute of content block
-                    # then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it.
-                    file_contents = [c for c in _content_block if c.get("type") == "file"]
-                    for file_content in file_contents:
-                        file_id = file_content.get("file", {}).get("file_id")
-                        if file_id:
-                            # Replace 'file' with 'file_id'
-                            file_content["file_id"] = file_id  # pyright: ignore[reportGeneralTypeIssues]  # legacy in-place rewrite of the block shape
-                            file_content.pop("file", None)
-        return messages
+        return [self._with_mistral_file_ids(m) for m in messages]
 
     def _add_reasoning_system_prompt_if_needed(
         self, messages: list[AllMessageValues], optional_params: dict
@@ -794,7 +921,16 @@ class MistralConfig(OpenAIGPTConfig):
         if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False):
             messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params)
 
-        upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"}
+        cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params)
+        withheld_params: Final = (
+            ("client_metadata", "prompt_cache_key")
+            if "prompt_cache_key" in cache_key_params
+            and self._prompt_cache_key_withheld(
+                model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params)
+            )
+            else ("client_metadata",)
+        )
+        upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params}
 
         # Call parent transform_request which handles _transform_messages
         return super().transform_request(

litellm/llms/oci/chat/transformation.py

@@ -21,6 +21,7 @@ from litellm._logging import verbose_logger
 from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
 from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
+from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@@ -428,6 +429,13 @@ class OCIChatConfig(BaseConfig):
             alias = param_map.get(key)
             if key == "reasoning_effort" and alias and not _model_supports_reasoning_effort(model):
                 if drop_params or litellm.drop_params:
+                    warn_dropped_reasoning(
+                        key,
+                        value,
+                        model,
+                        "the model map lists no reasoning_effort levels for it, so OCI would reject reasoningEffort.",
+                        drop_params_gated=True,
+                    )
                     continue
                 raise OCIError(
                     status_code=400,
@@ -451,6 +459,14 @@ class OCIChatConfig(BaseConfig):
                 if key == "n" and (value is None or value == 1):
                     continue
                 if drop_params or litellm.drop_params:
+                    if key == "reasoning_effort":
+                        warn_dropped_reasoning(
+                            key,
+                            value,
+                            model,
+                            "OCI's Cohere chat API has no reasoning effort field.",
+                            drop_params_gated=True,
+                        )
                     continue
                 raise OCIError(
                     status_code=400,

litellm/llms/ollama/chat/transformation.py

@@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_images_from_message,
     extract_reasoning_content,
 )
+from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@@ -29,7 +30,7 @@ from litellm.types.llms.openai import (
 )
 from litellm.types.utils import ModelResponse, ModelResponseStream
 
-from ..common_utils import OllamaError
+from ..common_utils import OllamaError, ollama_think
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -172,10 +173,7 @@ class OllamaChatConfig(BaseConfig):
                 if value.get("json_schema") and value["json_schema"].get("schema"):
                     optional_params["format"] = value["json_schema"]["schema"]
             if param == "reasoning_effort" and value is not None:
-                if model.startswith("gpt-oss"):
-                    optional_params["think"] = value
-                else:
-                    optional_params["think"] = value in {"low", "medium", "high"}
+                optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value))
             ### FUNCTION CALLING LOGIC ###
             # Ollama 0.4+ supports native tool calling - pass tools directly
             # and let Ollama handle model capability detection

litellm/llms/ollama/common_utils.py

@@ -8,11 +8,14 @@ from pydantic import BaseModel, TypeAdapter
 
 from litellm import verbose_logger
 from litellm._uuid import uuid
+from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.utils import ModelResponse, ModelResponseStream
 
 _PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object])
+_THINK_ON_EFFORTS: Final = frozenset({"low", "medium", "high"})
+_THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm maps this level to off."
 _DECODED_JSON: Final = TypeAdapter(object)
 
 
@@ -296,3 +299,12 @@ class OllamaModelInfo(BaseLLMModelInfo):
         Return the base model name for Ollama (no-op).
         """
         return model
+
+
+def ollama_think(model: str, reasoning_effort: object) -> object:
+    if model.startswith("gpt-oss"):
+        return reasoning_effort
+    think: Final = reasoning_effort in _THINK_ON_EFFORTS
+    if isinstance(reasoning_effort, str) and not think and reasoning_effort != "none":
+        warn_dropped_reasoning("reasoning_effort", reasoning_effort, model, _THINK_OFF_REASON, drop_params_gated=False)
+    return think

litellm/llms/ollama/completion/transformation.py

@@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
     async_inline_remote_media,
     inline_remote_image_urls,
 )
+from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@@ -43,7 +44,7 @@ from litellm.types.utils import (
     generate_id,
 )
 
-from ..common_utils import OllamaError, OllamaModelInfo, convert_image
+from ..common_utils import OllamaError, OllamaModelInfo, convert_image, ollama_think
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -249,10 +250,7 @@ class OllamaConfig(BaseConfig):
             elif param == "stop":
                 optional_params["stop"] = value
             elif param == "reasoning_effort" and value is not None:
-                if model.startswith("gpt-oss"):
-                    optional_params["think"] = value
-                else:
-                    optional_params["think"] = value in {"low", "medium", "high"}
+                optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value))
             elif param == "response_format" and isinstance(value, dict):
                 if value["type"] == "json_object":
                     optional_params["format"] = "json"

litellm/llms/openai/common_utils.py

@@ -32,6 +32,7 @@ import litellm
 from litellm.exceptions import error_response_without_credentials
 from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
+from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
 from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
@@ -165,17 +166,11 @@ _DECODED_JSON: Final = TypeAdapter(object)
 def drop_params_from_unprocessable_entity_error(
     e: openai.UnprocessableEntityError | httpx.HTTPStatusError,
     data: Mapping[str, object],
+    *,
+    model: str | None,
+    drop_params_gated: bool,
 ) -> dict[str, object]:
-    """
-    Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message.
-
-    Args:
-    e (UnprocessableEntityError): The UnprocessableEntityError exception
-    data (Dict[str, Any]): The original data dictionary containing all parameters
-
-    Returns:
-    Dict[str, Any]: A new dictionary with invalid parameters removed
-    """
+    """Return a copy of ``data`` without the params a 422 response names as invalid."""
     invalid_params: Final[list[str]] = []
     if isinstance(e, httpx.HTTPStatusError):
         error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json()))
@@ -203,6 +198,13 @@ def drop_params_from_unprocessable_entity_error(
                     invalid_params.append(error_dict["loc"][1])
 
     new_data: Final = {k: v for k, v in data.items() if k not in invalid_params}
+    warn_dropped_reasoning_params(
+        invalid_params,
+        data,
+        model,
+        "the provider answered 422 naming it an invalid param.",
+        drop_params_gated=drop_params_gated,
+    )
 
     return new_data
 

litellm/llms/openai/openai.py

@@ -27,6 +27,7 @@ import litellm
 from litellm import LlmProviders
 from litellm._logging import verbose_logger
 from litellm.constants import DEFAULT_MAX_RETRIES
+from litellm.exceptions import UnsupportedParamsError
 from litellm.files.types import FileContentStreamingResult
 from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -939,7 +940,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 except openai.UnprocessableEntityError as e:
                     ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
                     if litellm.drop_params is True or drop_params is True:
-                        inference_params = drop_params_from_unprocessable_entity_error(e, inference_params)
+                        inference_params = drop_params_from_unprocessable_entity_error(
+                            e, inference_params, model=model, drop_params_gated=True
+                        )
                     else:
                         raise e
                     # e.message
@@ -971,8 +974,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                         litellm.remove_index_from_tool_calls(messages=messages)
                     else:
                         raise e
-        except OpenAIError as e:
-            raise e
+        except (OpenAIError, UnsupportedParamsError):
+            raise
         except Exception as e:
             status_code: Final = getattr(e, "status_code", 500)
             error_headers = getattr(e, "headers", None)
@@ -1096,7 +1099,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             except openai.UnprocessableEntityError as e:
                 ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
                 if litellm.drop_params is True or drop_params is True:
-                    data = drop_params_from_unprocessable_entity_error(e, data)
+                    data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
                 else:
                     raise e
                 # e.message
@@ -1251,7 +1254,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             except openai.UnprocessableEntityError as e:
                 ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
                 if litellm.drop_params is True or drop_params is True:
-                    data = drop_params_from_unprocessable_entity_error(e, data)
+                    data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
                 else:
                     raise e
             except (

litellm/llms/openrouter/chat/transformation.py

@@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, cast
 import httpx
 from pydantic import ConfigDict, TypeAdapter
 
+from litellm.litellm_core_utils.core_helpers import finite_cost
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
@@ -225,8 +226,11 @@ class OpenrouterConfig(OpenAIGPTConfig):
                     additional_headers: Final = cast(  # cast-ok: preserve mapping operations on response metadata
                         dict[str, object], hidden_params["additional_headers"]
                     )
-                    if isinstance(response_cost, str | int | float):
-                        additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost)
+                    reported_cost: Final = (
+                        finite_cost(float(response_cost)) if isinstance(response_cost, str | int | float) else None
+                    )
+                    if reported_cost is not None:
+                        additional_headers["llm_provider-x-litellm-response-cost"] = reported_cost
         except Exception:
             # If we can't extract cost, continue without it - don't fail the response
             pass

litellm/llms/pass_through/typesafe_logging_handler.py

@@ -6,6 +6,7 @@ import httpx
 from pydantic import TypeAdapter, ValidationError
 
 import litellm
+from litellm.litellm_core_utils.core_helpers import finite_reported_cost
 from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.litellm_logging import (
@@ -100,9 +101,10 @@ class TypeSafePassthroughLoggingHandler:
             f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
         )
         pricing: Final = _pricing_for(candidate_model_keys)
+        reported_cost: Final = finite_reported_cost(usage.cost) if custom_llm_provider == "openrouter" else None
         response_cost: Final = (
-            usage.cost
-            if custom_llm_provider == "openrouter" and usage.cost is not None
+            reported_cost
+            if reported_cost is not None
             else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
         )
         usage_object: Final = Usage(

litellm/llms/perplexity/cost_calculator.py

@@ -6,6 +6,7 @@ Helper util for handling perplexity-specific cost calculation
 from datetime import datetime
 from typing import Final
 
+from litellm.litellm_core_utils.core_helpers import finite_reported_cost
 from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
 from litellm.types.utils import Usage
 from litellm.utils import get_model_info
@@ -34,10 +35,9 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
         total_cost = cost_info.get("total_cost")
     elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool):
         total_cost = float(cost_info)
-    if total_cost is not None:
-        # Return total cost as completion_cost (prompt_cost=0) since Perplexity
-        # doesn't break down by input/output in their cost object
-        return (0.0, float(total_cost))
+    reported_cost: Final = None if total_cost is None else finite_reported_cost(float(total_cost))
+    if reported_cost is not None:
+        return (0.0, reported_cost)
 
     ## FALLBACK: Calculate cost manually if Perplexity doesn't provide it
     ## GET MODEL INFO

litellm/llms/perplexity/responses/transformation.py

@@ -9,6 +9,7 @@ The only provider quirks:
 Ref: https://docs.perplexity.ai/api-reference/responses-post
 """
 
+from types import MappingProxyType
 from typing import Any, Final
 
 import httpx
@@ -17,6 +18,7 @@ from pydantic import TypeAdapter
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse
@@ -24,6 +26,9 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
 
 _DECODED_JSON: Final = TypeAdapter(object)
+PERPLEXITY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = MappingProxyType(
+    {**STANDARD_MODE_REASONING_FIELD_VALUES, "context": frozenset({"auto", "current_turn", "all_turns"})}
+)
 
 
 class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
@@ -127,3 +132,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
     def supports_native_websocket(self) -> bool:
         """Perplexity does not support native WebSocket for Responses API"""
         return False
+
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return PERPLEXITY_REASONING_FIELD_VALUES
+
+    def ignores_unsupported_reasoning_fields(self) -> bool:
+        return True

litellm/llms/sagemaker/chat/transformation.py

@@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import LlmProviders
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
-from ..common_utils import AWSEventStreamDecoder, SagemakerError
+from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -59,7 +59,12 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
         inference_component_name: Final = optional_params.get("model_id")
         if not isinstance(inference_component_name, str):
             return headers
-        return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name}
+        other_headers: Final = {
+            name: value
+            for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
+            if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower()
+        }
+        return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name}
 
     def transform_request(
         self,
@@ -76,9 +81,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
             litellm_params=litellm_params,
             headers=headers,
         )
-        container_request: Final = {
-            key: value for key, value in request.items() if key not in self.aws_authentication_params
-        }
+        header_only_params: Final = frozenset((*self.aws_authentication_params, "model_id"))
+        container_request: Final = {key: value for key, value in request.items() if key not in header_only_params}
         served_model_name: Final = litellm_params.get("hf_model_name")
         if not isinstance(served_model_name, str):
             return container_request

litellm/llms/sagemaker/common_utils.py

@@ -14,9 +14,10 @@ from typing_extensions import ReadOnly, Required, TypedDict
 from litellm import verbose_logger
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.utils import GenericStreamingChunk as GChunk
-from litellm.types.utils import StreamingChatCompletionChunk
+from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk
 
 STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
 
 
 @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
@@ -167,6 +168,36 @@ def get_sagemaker_response_stream_shape():
     return _load_sagemaker_response_stream_shape()
 
 
+INFERENCE_COMPONENT_HEADER: Final = "X-Amzn-SageMaker-Inference-Component"
+
+
+_SAGEMAKER_PROVIDERS: Final = frozenset(
+    provider.value for provider in (LlmProviders.SAGEMAKER, LlmProviders.SAGEMAKER_CHAT, LlmProviders.SAGEMAKER_NOVA)
+)
+
+
+def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None:
+    custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
+    if isinstance(custom_llm_provider, str) and custom_llm_provider:
+        return custom_llm_provider
+    model: Final = litellm_params.get("model")
+    return model.split("/", 1)[0] if isinstance(model, str) else None
+
+
+def deployment_inference_component(litellm_params: Mapping[str, object]) -> str | None:
+    headers: Final = litellm_params.get("extra_headers")
+    if _deployment_provider(litellm_params) not in _SAGEMAKER_PROVIDERS or not isinstance(headers, Mapping):
+        return None
+    return next(
+        (
+            value
+            for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
+            if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str)
+        ),
+        None,
+    )
+
+
 def with_inference_component_hint(error_message: str) -> str:
     if "Inference Component Name header is required" in error_message:
         return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`"

litellm/llms/sagemaker/completion/transformation.py

@@ -23,7 +23,7 @@ from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import ModelResponse, Usage
 from litellm.utils import token_counter
 
-from ..common_utils import SagemakerError
+from ..common_utils import STR_KEYED_MAPPING, SagemakerError
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -153,7 +153,9 @@ class SagemakerConfig(BaseConfig):
         litellm_params: dict,
         headers: dict,
     ) -> dict:
-        inference_params: Final = optional_params.copy()
+        inference_params: Final = {
+            key: value for key, value in STR_KEYED_MAPPING.validate_python(optional_params).items() if key != "model_id"
+        }
         stream: Final = inference_params.pop("stream", False)
         data: Final[dict] = {"parameters": inference_params}
         if stream is True:

litellm/llms/subscription_accounts.py

@@ -0,0 +1,37 @@
+"""
+Providers whose requests are covered by a flat subscription, each with the way to read
+the account id of the login a deployment is signed into.
+"""
+
+from collections.abc import Callable, Mapping
+from types import MappingProxyType
+from typing import Final
+
+from litellm.llms.chatgpt.authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
+from litellm.secret_managers.main import UnresolvableSecretReference, resolve_local_oidc_reference
+from litellm.utils import with_named_credential
+
+
+def _chatgpt_account_id(litellm_params: Mapping[str, object]) -> str | None:
+    params: Final = with_named_credential(litellm_params)
+    auth_file: Final = get_chatgpt_auth_file(params)
+    if auth_file:
+        return get_cached_authenticator(auth_file).read_account_id()
+    api_key: Final = params.get("api_key")
+    if not isinstance(api_key, str) or not api_key:
+        return Authenticator().read_account_id()
+    match resolve_local_oidc_reference(api_key):
+        case UnresolvableSecretReference():
+            return None
+        case str() as access_token:
+            return Authenticator().read_account_id(access_token)
+
+
+SUBSCRIPTION_ACCOUNT_ID_RESOLVERS: Final[Mapping[str, Callable[[Mapping[str, object]], str | None]]] = MappingProxyType(
+    {"chatgpt": _chatgpt_account_id}
+)
+
+
+def resolve_subscription_account_id(custom_llm_provider: str, litellm_params: Mapping[str, object]) -> str | None:
+    resolver: Final = SUBSCRIPTION_ACCOUNT_ID_RESOLVERS.get(custom_llm_provider)
+    return None if resolver is None else resolver(litellm_params)

litellm/llms/tencent/chat/transformation.py

@@ -9,12 +9,15 @@ from typing import Final, TypedDict
 from typing_extensions import ReadOnly
 
 import litellm
+from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
 from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
 from litellm.secret_managers.main import get_secret_str
 from litellm.utils import supports_reasoning
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
+_ADAPTIVE_BUDGET_REASON: Final = "it is not an integer, so litellm sends adaptive thinking with no budget."
+
 
 class ThinkingPayload(TypedDict, total=False):
     """Tencent TokenHub `thinking` object.
@@ -102,6 +105,10 @@ class TencentChatConfig(OpenAIGPTConfig):
         if isinstance(budget, int):
             coerced_with_budget: Final[ThinkingPayload] = {"type": "adaptive", "budget_tokens": budget}
             return coerced_with_budget
+        if budget is not None:
+            warn_dropped_reasoning(
+                "thinking.budget_tokens", budget, model, _ADAPTIVE_BUDGET_REASON, drop_params_gated=False
+            )
         coerced: Final[ThinkingPayload] = {"type": "adaptive"}
         return coerced
 

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc
 from litellm.litellm_core_utils.prompt_templates.factory import (
     encode_tool_call_id_with_signature,
 )
+from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -201,6 +202,13 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr
 
 
 _SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map"
+_NON_INTEGER_BUDGET_REASON: Final = (
+    "Gemini takes budget_tokens only as an integer, so it uses its default thinking budget."
+)
+_LEVEL_ONLY_BUDGET_REASON: Final = (
+    "Gemini 3 and later set reasoning by thinking level, not budget, so litellm does not send the budget. "
+    "Send reasoning_effort to pick a level."
+)
 _sampling_deprecation_lock: Final = threading.Lock()
 
 
@@ -1466,6 +1474,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                     params["includeThoughts"] = False
                 else:
                     params["includeThoughts"] = sends_thoughts
+                    if thinking_budget is not None:
+                        warn_dropped_reasoning(
+                            "thinking.budget_tokens",
+                            thinking_budget,
+                            model,
+                            _LEVEL_ONLY_BUDGET_REASON,
+                            drop_params_gated=False,
+                        )
                     # Follow provider defaults unless explicitly opted into legacy behavior.
                     if litellm.enable_gemini_default_thinking_level_low is True:
                         params["thinkingLevel"] = (
@@ -1478,8 +1494,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             # For older Gemini models, use thinkingBudget
             if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget):
                 params["includeThoughts"] = sends_thoughts
-            if thinking_budget is not None and isinstance(thinking_budget, int):
+            if isinstance(thinking_budget, int):
                 params["thinkingBudget"] = thinking_budget
+            elif thinking_budget is not None:
+                warn_dropped_reasoning(
+                    "thinking.budget_tokens",
+                    thinking_budget,
+                    model,
+                    _NON_INTEGER_BUDGET_REASON,
+                    drop_params_gated=False,
+                )
 
         return params
 

litellm/llms/vertex_ai/realtime/transformation.py

@@ -39,8 +39,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
     (they require async I/O) and injected at construction time.
     """
 
-    def __init__(self, access_token: str, project: str, location: str) -> None:
-        super().__init__()
+    def __init__(self, access_token: str, project: str, location: str, drop_params: bool = False) -> None:
+        super().__init__(drop_params=drop_params)
         self._access_token = access_token
         self._project = project
         self._location = location
@@ -51,6 +51,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
     def _rejects_vad_tuning_with_detection_off(self) -> bool:
         return False
 
+    def _rejects_undocumented_voices(self, model: str) -> bool:
+        return True
+
     # ------------------------------------------------------------------
     # URL
     # ------------------------------------------------------------------
@@ -163,7 +166,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
         # settings would be silently dropped because ``map_openai_params`` only
         # recognises the flat OpenAI-beta key names.
         session_params = self._normalize_session_payload_for_mapping(session_params)
-        setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params)
+        setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params, model=model)
 
         # Use full Vertex AI model path
         setup_config["model"] = self._vertex_model_path(model)
@@ -177,15 +180,16 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
         # ``silence_duration_ms``). ``map_automatic_turn_detection`` sets
         # ``disabled=True`` whenever ``create_response`` is absent or
         # ``False``. Force ``disabled=False`` only when the client did
-        # not explicitly request ``create_response: False`` — that path
+        # not explicitly request ``create_response: False`` or
+        # ``turn_detection: null`` — the first of those paths
         # is how transcription guardrails suppress automatic responses,
         # and overriding it here would silently bypass the guardrail.
         # Vertex Live has no "VAD on, no auto-response" mode, so callers
         # that need that behaviour must accept that VAD is off.
         client_turn_detection: Final = session_params.get("turn_detection")
         client_disabled_auto_response: Final = (
-            isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False
-        )
+            "turn_detection" in session_params and client_turn_detection is None
+        ) or (isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False)
         realtime_input_config: Final = setup_config.setdefault("realtimeInputConfig", {})
         automatic_detection: Final = realtime_input_config.setdefault("automaticActivityDetection", {})
         if not client_disabled_auto_response:
@@ -261,4 +265,5 @@ def vertex_realtime_config(
         access_token=access_token,
         project=project,
         location=VertexBase.get_vertex_region(vertex_region=location, model=model),
+        drop_params=drop_params,
     )

litellm/llms/xai/responses/transformation.py

@@ -10,6 +10,10 @@ from litellm._logging import verbose_logger
 from litellm.constants import XAI_API_BASE
 from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.base_llm.responses.transformation import (
+    STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
+    ReasoningFieldValues,
+)
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd
 from litellm.secret_managers.main import get_secret_str
@@ -290,3 +294,9 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def supports_native_websocket(self) -> bool:
         """XAI does not support native WebSocket for Responses API"""
         return False
+
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
+
+    def ignores_unsupported_reasoning_fields(self) -> bool:
+        return True

litellm/main.py

@@ -681,7 +681,7 @@ async def acompletion(
         - The `completion` function is called using `run_in_executor` to execute synchronously in the event loop.
         - If `stream` is True, the function returns an async generator that yields completion lines.
     """
-    fallbacks = kwargs.get("fallbacks", None)
+    fallbacks: Final = kwargs.get("fallbacks", None) or litellm.model_fallbacks
     mock_timeout = kwargs.get("mock_timeout", None)
 
     if mock_timeout is True:
@@ -697,6 +697,7 @@ async def acompletion(
     loop: Final = asyncio.get_event_loop()
     custom_llm_provider = kwargs.get("custom_llm_provider", None)
     _ = _resolve_control_options(kwargs, model)
+    caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs})  # cast-ok: acompletion takes untyped kwargs
 
     ## PROMPT MANAGEMENT HOOKS ##
     #########################################################
@@ -715,15 +716,34 @@ async def acompletion(
         custom_llm_provider=cast(str | None, custom_llm_provider),  # cast-ok: read from untyped kwargs
         tools=tools,
         enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")),  # cast-ok: untyped kwargs
-        api_base=kwargs.get("api_base") or base_url,
+        api_base=base_url or kwargs.get("api_base"),
     )
 
-    if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
-        litellm_logging_obj.should_run_prompt_management_hooks(
-            prompt_id=kwargs.get("prompt_id", None),
+    prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None))  # cast-ok: read from untyped kwargs
+    prompt_management_logger: Final = (
+        litellm_logging_obj.get_custom_logger_for_prompt_management(
+            model=model,
+            non_default_params=kwargs,
+            tools=tools,
+            prompt_id=prompt_id,
+            dynamic_callback_params=litellm_logging_obj.standard_callback_dynamic_params,
+        )
+        if isinstance(litellm_logging_obj, LiteLLMLoggingObj)
+        and litellm_logging_obj.should_run_prompt_management_hooks(
+            prompt_id=prompt_id,
             non_default_params=kwargs,
             tools=tools,
         )
+        else None
+    )
+    each_fallback_writes_its_own_cache_dialect: Final = fallbacks is not None and isinstance(
+        prompt_management_logger, AnthropicCacheControlHook
+    )
+
+    if (
+        isinstance(litellm_logging_obj, LiteLLMLoggingObj)
+        and prompt_management_logger is not None
+        and not each_fallback_writes_its_own_cache_dialect
     ):
         (
             model,
@@ -733,8 +753,9 @@ async def acompletion(
             model=model,
             messages=messages,
             non_default_params=kwargs,
-            prompt_id=kwargs.get("prompt_id", None),
+            prompt_id=prompt_id,
             prompt_variables=kwargs.get("prompt_variables", None),
+            prompt_management_logger=prompt_management_logger,
             tools=tools,
             prompt_label=kwargs.get("prompt_label", None),
             prompt_version=kwargs.get("prompt_version", None),
@@ -826,9 +847,10 @@ async def acompletion(
             litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
         )
 
-    fallbacks = fallbacks or litellm.model_fallbacks
     if fallbacks is not None:
-        response = await async_completion_with_fallbacks(**completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs})
+        response = await async_completion_with_fallbacks(
+            **completion_kwargs, kwargs={"fallbacks": fallbacks, **caller_kwargs}
+        )
         if response is None:
             raise Exception(
                 "No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details."
@@ -5765,7 +5787,7 @@ def completion(
         custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")),  # cast-ok: untyped kwargs
         tools=tools,
         enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")),  # cast-ok: untyped kwargs
-        api_base=kwargs.get("api_base") or base_url,
+        api_base=base_url or kwargs.get("api_base"),
     )
 
     if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
@@ -6199,6 +6221,7 @@ def completion(
                     PROVIDER_AFFINITY_HEADER_KWARG_KEY,
                     "fireworks_forward_user_id",
                     "responses_api_version",
+                    "allowed_openai_params",
                 )
                 if key in kwargs
             },
@@ -6266,6 +6289,9 @@ def completion(
         # reasoningSummary/reasoning_summary without tools (AI SDK) that the first
         # (early) check doesn't cover.
         _reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params)
+        typed_deployment_model_info: Final = str_keyed_mapping(
+            cast(object, model_info)  # cast-ok: model_info is the untyped dict kwarg
+        )
         if responses_api_model_info.get("mode") != "responses":
             responses_api_model_info, model = responses_api_bridge_check(
                 model=model,
@@ -6275,9 +6301,7 @@ def completion(
                 reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")),
                 reasoning_summary=_reasoning_summary_for_bridge,
                 api_base=api_base,
-                deployment_model_info=str_keyed_mapping(
-                    cast(object, model_info)  # cast-ok: model_info is the untyped dict kwarg
-                ),
+                deployment_model_info=typed_deployment_model_info,
             )
 
         # Use base_model (the true underlying model) for Azure model-type
@@ -6289,19 +6313,28 @@ def completion(
 
             optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params)
 
-            if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict):
-                from litellm.responses.utils import openai_bridge_reasoning
+            if isinstance(resolved_reasoning_effort, dict):
+                from litellm.responses.utils import (
+                    RefusedReasoningFields,
+                    bridge_reasoning,
+                    raise_refused_reasoning_fields,
+                )
 
-                optional_params["reasoning_effort"] = openai_bridge_reasoning(
+                bridged_reasoning: Final = bridge_reasoning(
                     requested=resolved_reasoning_effort,
                     effort_kept="reasoning_effort" in optional_params,
                     summary_alias=rs_val,
                     model=model,
+                    base_model=base_model if isinstance(base_model, str) else None,
+                    custom_llm_provider=custom_llm_provider,
+                    api_base=api_base,
+                    deployment_model_info=typed_deployment_model_info,
                     drop_params=drop_unsupported_reasoning,
                     additional_drop_params=routing_kwargs.get("additional_drop_params"),
                 )
-            elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort:
-                optional_params["reasoning_effort"] = resolved_reasoning_effort
+                if isinstance(bridged_reasoning, RefusedReasoningFields):
+                    raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider)
+                optional_params["reasoning_effort"] = bridged_reasoning
             elif rs_val is not None:
                 eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort)
                 if isinstance(eff, dict):

litellm/models/__init__.py

@@ -30,6 +30,7 @@ from litellm.models.organization_membership import LiteLLM_OrganizationMembershi
 from litellm.models.project import LiteLLM_ProjectTable
 from litellm.models.skills import LiteLLM_SkillsTable
 from litellm.models.spend_logs import LiteLLM_ErrorLogs, LiteLLM_SpendLogs
+from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
 from litellm.models.tag import LiteLLM_TagTable
 from litellm.models.team import LiteLLM_TeamTable
 from litellm.models.team_membership import LiteLLM_TeamMembership
@@ -59,6 +60,7 @@ __all__ = [
     "LiteLLM_ProxyModelTable",
     "LiteLLM_SkillsTable",
     "LiteLLM_SpendLogs",
+    "LiteLLM_SubscriptionAccountTable",
     "LiteLLM_TagTable",
     "LiteLLM_TeamMemberTable",
     "LiteLLM_TeamMembership",

litellm/models/subscription_account.py

@@ -0,0 +1,28 @@
+"""
+Subscription account table model.
+
+Canonical definition for ``litellm_subscriptionaccounttable``: the operator-entered
+recurring fee of a provider account whose requests are covered by a flat subscription.
+"""
+
+from datetime import datetime
+
+from pydantic import ConfigDict
+
+from litellm.types.llms.base import LiteLLMPydanticObjectBase
+
+
+class LiteLLM_SubscriptionAccountTable(LiteLLMPydanticObjectBase):
+    model_config = ConfigDict(protected_namespaces=(), frozen=True)
+
+    subscription_account_id: str
+    custom_llm_provider: str
+    account_id: str
+    label: str | None = None
+    monthly_fee: float
+    currency: str
+    billing_period_start: str
+    created_at: datetime
+    created_by: str
+    updated_at: datetime
+    updated_by: str

litellm/proxy/_lazy_openapi_snapshot.json

@@ -1648,7 +1648,7 @@
       },
       "/access_group/{access_group}/update": {
         "put": {
-          "description": "Update an access group's model names.\n\nThis will:\n1. Remove the access group from all current deployments\n2. Add the access group to all deployments for the new model_names list\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n  -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n  -H 'Content-Type: application/json' \\\n  -d '{\n    \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n  }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found",
+          "description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n  -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n  -H 'Content-Type: application/json' \\\n  -d '{\n    \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n  }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit holds the group or one of its models for 30 s",
           "operationId": "update_access_group_access_group__access_group__update_put",
           "parameters": [
             {

litellm/proxy/_types.py

@@ -667,6 +667,10 @@ class LiteLLMRoutes(enum.Enum):
             "/team/spend/by_user",
             # gateway request counts (SGR); deployment-wide, admin-only
             "/gateway/daily/activity",
+            "/subscription_accounts/usage",
+            "/subscription_accounts/new",
+            "/subscription_accounts/update",
+            "/subscription_accounts/delete",
             # model
             "/model/new",
             "/model/update",

litellm/proxy/auth/auth_checks.py

@@ -84,6 +84,7 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
+    AgentAccessGroupCeiling,
     CeilingResolver,
     resolve_agent_access_group_ceiling,
 )
@@ -1684,6 +1685,7 @@ def get_actual_routes(allowed_routes: list) -> list:
 
 KEY_END_USER_BUDGET_ID_METADATA_FIELD: Final = "end_user_budget_id"
 _KEY_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
+_OBJECTS_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
 
 
 def get_key_end_user_budget_id(key_metadata: Mapping[str, object] | None) -> str | None:
@@ -5085,6 +5087,36 @@ async def can_agent_call_model(
     )
 
 
+async def _agent_access_group_ceilings(
+    valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver
+) -> tuple[AgentAccessGroupCeiling, ...]:
+    if not valid_token.agent_id:
+        return ()
+
+    from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings
+
+    managed: Final = managed_agent_policy(valid_token)
+    if managed is not None:
+        return await resolve_managed_agent_ceilings(managed)
+    unmanaged: Final = await resolve_ceiling(valid_token.agent_id)
+    return (unmanaged,) if unmanaged is not None else ()
+
+
+def _managed_agent_model_grants(valid_token: UserAPIKeyAuth) -> frozenset[str]:
+    managed: Final = managed_agent_policy(valid_token)
+    granted: Final = (managed.object_permission or {}).get("models") if managed is not None else None
+    if not isinstance(granted, (list, tuple)):
+        return frozenset()
+    return frozenset(name for name in _OBJECTS_ADAPTER.validate_python(granted) if isinstance(name, str))
+
+
+async def agent_model_grants(
+    valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling
+) -> frozenset[str]:
+    ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling)
+    return _managed_agent_model_grants(valid_token).union(*(ceiling.models for ceiling in ceilings))
+
+
 async def _check_agent_access_group_model_access(
     model: str | list[str] | None,  # mutable-ok: _can_object_call_model and the client message helper take list[str]
     valid_token: UserAPIKeyAuth | None,
@@ -5092,20 +5124,9 @@ async def _check_agent_access_group_model_access(
     resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling,
 ) -> Literal[True]:
     """Attached groups naming no model deny every model; the empty allowlist in ``_can_object_call_model`` allows."""
-    if not model or valid_token is None or not valid_token.agent_id:
+    if not model or valid_token is None:
         return True
-
-    from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings
-
-    managed: Final = managed_agent_policy(valid_token)
-    unmanaged: Final = await resolve_ceiling(valid_token.agent_id) if managed is None else None
-    ceilings: Final = (
-        await resolve_managed_agent_ceilings(managed)
-        if managed is not None
-        else (unmanaged,)
-        if unmanaged is not None
-        else ()
-    )
+    ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling)
     dispatched: Final = _resolve_team_alias(
         model, team_model_aliases_for_auth_check(valid_token), valid_token.team_id, llm_router
     )

litellm/proxy/auth/handle_jwt.py

@@ -2199,15 +2199,19 @@ class JWTAuthManager:
         # not mistaken for claimless and left with stale DB memberships the fallback
         # could later attribute. With the flag off, keep the upstream plural-only
         # reconciliation so existing deployments are unchanged.
-        jwt_team_ids: Final = set(
-            jwt_handler.get_all_jwt_team_ids(jwt_valid_token)
-            if jwt_handler.litellm_jwtauth.fallback_to_db_teams
-            else jwt_handler.get_team_ids_from_jwt(jwt_valid_token)
+        claim_team_ids: Final = tuple(
+            dict.fromkeys(
+                jwt_handler.get_all_jwt_team_ids(jwt_valid_token)
+                if jwt_handler.litellm_jwtauth.fallback_to_db_teams
+                else jwt_handler.get_team_ids_from_jwt(jwt_valid_token)
+            )
         )
-        existing_teams: Final = set(user_object.teams or [])
-        teams_to_add: Final = jwt_team_ids - existing_teams
+        jwt_team_ids: Final = frozenset(claim_team_ids)
+        existing_team_order: Final = tuple(dict.fromkeys(user_object.teams or []))
+        existing_teams: Final = frozenset(existing_team_order)
+        teams_to_add: Final = tuple(team_id for team_id in claim_team_ids if team_id not in existing_teams)
         preserve_db_teams_without_claims: Final = jwt_handler.litellm_jwtauth.fallback_to_db_teams and not jwt_team_ids
-        teams_to_remove: Final = set() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids
+        teams_to_remove: Final = frozenset() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids
         if teams_to_add or teams_to_remove:
             from litellm.proxy.management_endpoints.scim.scim_v2 import (
                 patch_team_membership,
@@ -2218,7 +2222,10 @@ class JWTAuthManager:
                 teams_ids_to_add_user_to=list(teams_to_add),
                 teams_ids_to_remove_user_from=list(teams_to_remove),
             )
-            user_object.teams = list(jwt_team_ids)
+            user_object.teams = [
+                *(team_id for team_id in existing_team_order if team_id not in teams_to_remove),
+                *teams_to_add,
+            ]
             if user_api_key_cache is not None:
                 await user_api_key_cache.async_set_cache(
                     key=user_object.user_id,

litellm/proxy/common_request_processing.py

@@ -60,6 +60,7 @@ from litellm.litellm_core_utils.bug_report import (
     strip_bug_report_notice,
 )
 from litellm.litellm_core_utils.core_helpers import (
+    finite_cost,
     get_or_create_metadata_bucket,
     independent_snapshot,
     is_expected_client_error,
@@ -140,6 +141,7 @@ from litellm.proxy.utils import (  # noqa: F401  # legacy module exports
     _check_and_merge_model_level_guardrails,  # pyright: ignore[reportPrivateUsage,reportUnusedImport]  # backwards-compatible package export
     check_and_merge_model_level_guardrails,
 )
+from litellm.responses.mcp.parked_stream_dispatch import in_flight_round_biller, parked_success
 from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
 from litellm.router import Router
 from litellm.router_api import RoutingContext
@@ -535,6 +537,16 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
     return await _dispatch_disconnect_billing_event(logging_obj, partial_response)
 
 
+async def _bill_in_flight_round_on_disconnect(request_data: Mapping[str, object]) -> None:
+    bill_in_flight_round: Final = in_flight_round_biller(request_data.get("litellm_logging_obj"))
+    if bill_in_flight_round is None:
+        return
+    try:
+        await bill_in_flight_round()
+    except Exception as e:  # noqa: BLE001  # disconnect billing is best-effort; never break stream teardown
+        verbose_proxy_logger.debug("Failed to bill the in-flight round after client disconnect: %s", e)
+
+
 async def _dispatch_disconnect_billing_event(logging_obj: LiteLLMLoggingObj, partial_response: object) -> bool:
     try:
         await logging_obj.dispatch_success_handlers(
@@ -1801,14 +1813,16 @@ def _get_cost_breakdown_from_logging_obj(
 
 
 def _header_value(value: float | None) -> str | None:
-    return str(value) if value is not None else None
+    return str(value) if finite_cost(value) is not None else None
 
 
 def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float | str:
     if hidden_params.get("cache_hit") is True:
         return 0.0
     response_cost: Final = hidden_params.get("response_cost")
-    return response_cost if isinstance(response_cost, (int, float, str)) and response_cost else ""
+    if isinstance(response_cost, str):
+        return response_cost
+    return response_cost if isinstance(response_cost, int | float) and finite_cost(response_cost) else ""
 
 
 def cost_headers(
@@ -2083,6 +2097,13 @@ class _ClientStreamFailureReport:
             )
 
 
+def _metadata_logged_on(logged_on: LiteLLMLoggingObj) -> Mapping[str, object]:
+    logged_params: Final = object_mapping(logged_on.model_call_details.get("litellm_params")) or MappingProxyType({})
+    return MappingProxyType(
+        {key: logged_params[key] for key in ("metadata", "litellm_metadata") if key in logged_params}
+    )
+
+
 class ProxyBaseLLMRequestProcessing:
     def __init__(self, data: dict, served_model: str | None = None):
         self.data = data
@@ -4151,11 +4172,21 @@ class ProxyBaseLLMRequestProcessing:
             _captured_user_api_key_dict: Final = user_api_key_dict
             _captured_logging_obj: Final = logging_obj
 
-            async def _on_deferred_stream_complete(assembled_response: object, cache_hit: object) -> None:
+            async def _on_deferred_stream_complete(
+                assembled_response: object, cache_hit: object, logged_on: LiteLLMLoggingObj | None = None
+            ) -> None:
                 await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails(
-                    captured_data=_captured_data,
+                    captured_data=_captured_data
+                    if logged_on is None
+                    else {
+                        **_captured_data,
+                        **{
+                            key: value for key, value in _metadata_logged_on(logged_on).items() if key in _captured_data
+                        },
+                        "litellm_logging_obj": logged_on,
+                    },
                     captured_user_api_key_dict=_captured_user_api_key_dict,
-                    captured_logging_obj=_captured_logging_obj,
+                    captured_logging_obj=_captured_logging_obj if logged_on is None else logged_on,
                     assembled_response=assembled_response,
                     cache_hit=cache_hit,
                 )
@@ -4180,14 +4211,8 @@ class ProxyBaseLLMRequestProcessing:
                     GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
                 case (logging_coroutine,) if asyncio.iscoroutine(logging_coroutine):
                     GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine)
-                case (assembled_response, cache_hit):
-                    await _as_success_dispatcher(_captured_native_logging_obj).dispatch_success_handlers(
-                        assembled_response,
-                        cache_hit=cache_hit,
-                        start_time=None,
-                        end_time=None,
-                        prefer_async_handlers=True,
-                    )
+                case _ if (success := parked_success(args)) is not None:
+                    await success.dispatch(_captured_native_logging_obj)
                 case _:
                     verbose_proxy_logger.error(
                         "Deferred stream logging dropped: unexpected stored args shape %s",
@@ -4487,7 +4512,7 @@ class ProxyBaseLLMRequestProcessing:
         """
         if isinstance(chunk, dict):
             # Use safe_dumps for proper JSON serialization with circular reference detection
-            chunk_str: Final = safe_dumps(chunk)
+            chunk_str: Final = safe_dumps(chunk, non_finite_as_null=True)
             return f"{STREAM_SSE_DATA_PREFIX}{chunk_str}\n\n"
         else:
             return chunk
@@ -4524,6 +4549,7 @@ class ProxyBaseLLMRequestProcessing:
                     client_disconnected,
                 )
             if recorded_client_disconnect:
+                await _bill_in_flight_round_on_disconnect(_REQUEST_FIELDS.validate_python(request_data))
                 deferred_stream_logging_armed: Final = _deferred_stream_logging_is_armed(request_data)
                 ProxyLogging.fire_deferred_stream_logging(request_data)
                 # A disconnect-time success event (the deferred-guardrail flush
@@ -4862,7 +4888,11 @@ class ProxyBaseLLMRequestProcessing:
                             obj, model_name, litellm_logging_obj
                         )
                         if maybe_modified is not None:
-                            lines[idx] = "data: " + safe_dumps(maybe_modified) + ("\r" if ln.endswith("\r") else "")
+                            lines[idx] = (
+                                "data: "
+                                + safe_dumps(maybe_modified, non_finite_as_null=True)
+                                + ("\r" if ln.endswith("\r") else "")
+                            )
                             return "\n".join(lines)
             return None
         except Exception:

litellm/proxy/common_utils/config_sync_pubsub.py

@@ -2,9 +2,12 @@ import asyncio
 import json
 import random
 import time
-from collections.abc import Awaitable, Callable
+from collections.abc import AsyncIterator, Awaitable, Callable
 from dataclasses import asdict, dataclass
-from typing import TYPE_CHECKING, Final, Protocol, cast  # noqa: TID251  # untyped prisma/redis boundary needs cast
+from itertools import chain
+from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast  # noqa: TID251  # untyped prisma/redis boundary
+
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_proxy_logger
 from litellm.repositories.prisma_protocols import RowT_co, TableActions
@@ -106,24 +109,69 @@ _pubsub_capable_client: Final = pubsub_capable_client
 @dataclass(frozen=True, slots=True)
 class _ConfigChangeMessage:
     object_type: str
+    model_ids: tuple[str, ...] = ()
 
 
-def _config_change_message_json(object_type: str) -> str:
-    return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type)))
+@dataclass(frozen=True, slots=True)
+class _PubSubEnvelope:
+    data: str | bytes
 
 
-async def publish_config_change(redis_cache: "RedisCache | None", object_type: str) -> None:
+class _WrittenModelRow(BaseModel):
+    model_config = ConfigDict(from_attributes=True, frozen=True)
+
+    model_id: str
+
+
+_CONFIG_CHANGE_MESSAGE: Final = TypeAdapter(_ConfigChangeMessage)
+_PUBSUB_ENVELOPE: Final = TypeAdapter(_PubSubEnvelope)
+_MODEL_TABLE_NAME: Final = "litellm_proxymodeltable"
+_TARGETED_WRITE_ACTION_NAMES: Final[frozenset[str]] = frozenset({"create", "update", "upsert"})
+
+PublishConfigChange: TypeAlias = Callable[[str, tuple[str, ...]], Awaitable[None]]
+
+
+def _config_change_message_json(object_type: str, model_ids: tuple[str, ...]) -> str:
+    return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type, model_ids=model_ids)))
+
+
+def parse_config_change(message: object) -> _ConfigChangeMessage | None:
+    try:
+        payload: Final = (
+            message if isinstance(message, (str, bytes)) else _PUBSUB_ENVELOPE.validate_python(message).data
+        )
+        return _CONFIG_CHANGE_MESSAGE.validate_json(payload)
+    except ValidationError:
+        return None
+
+
+def written_model_ids(object_type: str, action_name: str, result: object) -> tuple[str, ...]:
+    if object_type != _MODEL_TABLE_NAME or action_name not in _TARGETED_WRITE_ACTION_NAMES:
+        return ()
+    try:
+        return (_WrittenModelRow.model_validate(result).model_id,)
+    except ValidationError:
+        return ()
+
+
+async def publish_config_change(
+    redis_cache: "RedisCache | None", object_type: str, model_ids: tuple[str, ...] = ()
+) -> None:
     if redis_cache is None:
         return
     try:
         client: Final = pubsub_capable_client(redis_cache)
-        await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type))
+        await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type, model_ids))
     except Exception as e:  # noqa: BLE001  # best-effort publish; writes must never fail on redis errors
         verbose_proxy_logger.warning("config sync publish for %s failed: %s", object_type, e)
 
 
-async def publish_config_change_for_object_type(object_type: str) -> None:
-    await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type)
+async def publish_config_change_for_object_type(object_type: str, model_ids: tuple[str, ...] = ()) -> None:
+    await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type, model_ids=model_ids)
+
+
+async def publish_model_rows_written(model_ids: tuple[str, ...]) -> None:
+    await publish_config_change_for_object_type(_MODEL_TABLE_NAME, model_ids)
 
 
 async def publish_config_param_change(param_name: str) -> None:
@@ -139,7 +187,7 @@ async def publish_config_param_change(param_name: str) -> None:
 class _PublishOnWriteActions:
     __slots__ = ("_actions", "_object_type", "_publish")
 
-    def __init__(self, actions: object, object_type: str, publish: Callable[[str], Awaitable[None]]) -> None:
+    def __init__(self, actions: object, object_type: str, publish: PublishConfigChange) -> None:
         self._actions = actions
         self._object_type = object_type
         self._publish = publish
@@ -157,7 +205,7 @@ class _PublishOnWriteActions:
             **kwargs: object,  # kwargs-ok: transparent passthrough to untyped prisma action
         ) -> object:
             result: Final = await write_action(*args, **kwargs)
-            await publish(object_type)
+            await publish(object_type, written_model_ids(object_type, name, result))
             return result
 
         return _write_then_publish
@@ -166,7 +214,7 @@ class _PublishOnWriteActions:
 def wrap_table_actions_for_config_sync(
     actions: "TableActions[RowT_co]",
     table_name: str,
-    publish: Callable[[str], Awaitable[None]] = publish_config_change_for_object_type,
+    publish: PublishConfigChange = publish_config_change_for_object_type,
 ) -> "TableActions[RowT_co]":
     if table_name not in _CONFIG_SYNCED_TABLE_NAMES:
         return actions
@@ -174,8 +222,14 @@ def wrap_table_actions_for_config_sync(
     return cast("TableActions[RowT_co]", wrapped)  # cast-ok: dynamic write-through proxy keeps the wrapped row type
 
 
+def _published_model_ids(messages: tuple[object, ...]) -> tuple[str, ...]:
+    changes: Final = (parse_config_change(message) for message in messages)
+    return tuple(dict.fromkeys(chain.from_iterable(change.model_ids for change in changes if change is not None)))
+
+
 class ConfigSyncSubscriber:
     __slots__ = (
+        "_apply_model_ids",
         "_backoff_initial_seconds",
         "_backoff_max_seconds",
         "_debounce_seconds",
@@ -195,6 +249,7 @@ class ConfigSyncSubscriber:
         self,
         redis_cache: "RedisCache",
         resync_callbacks: tuple[Callable[[], Awaitable[None]], ...],
+        apply_model_ids: Callable[[tuple[str, ...]], Awaitable[None]] | None = None,
         debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS,
         jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS,
         min_resync_interval_seconds: float = CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS,
@@ -206,6 +261,7 @@ class ConfigSyncSubscriber:
     ) -> None:
         self._redis_cache = redis_cache
         self._resync_callbacks = resync_callbacks
+        self._apply_model_ids = apply_model_ids
         self._debounce_seconds = debounce_seconds
         self._jitter_max_seconds = jitter_max_seconds
         self._min_resync_interval_seconds = min_resync_interval_seconds
@@ -266,32 +322,64 @@ class ConfigSyncSubscriber:
             message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=_POLL_TIMEOUT_SECONDS)
             if message is None:
                 continue
+            await self._apply(pubsub, message)
             await self._debounce_and_resync(pubsub)
 
-    async def _debounce_and_resync(self, pubsub: _ConfigSyncPubSub) -> None:
-        await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds))
-        await self._wait_for_min_resync_interval()
+    async def _debounce_and_resync(self, pubsub: ConfigSyncPubSub) -> None:
+        await self._apply_until(
+            pubsub, self._monotonic() + self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)
+        )
+        await self._apply_until(pubsub, self._next_resync_at())
         await self._drain_pending(pubsub)
         await self._run_resync_callbacks()
         self._last_resync_at = self._monotonic()
 
-    async def _wait_for_min_resync_interval(self) -> None:
+    def _next_resync_at(self) -> float:
         if self._last_resync_at is None:
-            return
-        seconds_until_next_resync = self._min_resync_interval_seconds - (self._monotonic() - self._last_resync_at)
-        if seconds_until_next_resync <= 0:
-            return
-        verbose_proxy_logger.debug(
-            "config sync resync throttled for %.1fs to cap fleet-wide reload rate",
-            seconds_until_next_resync,
-        )
-        await self._sleep(seconds_until_next_resync)
+            return self._monotonic()
+        next_resync_at: Final = self._last_resync_at + self._min_resync_interval_seconds
+        throttled_for: Final = next_resync_at - self._monotonic()
+        if throttled_for > 0:
+            verbose_proxy_logger.debug(
+                "config sync resync throttled for %.1fs to cap fleet-wide reload rate",
+                throttled_for,
+            )
+        return next_resync_at
+
+    async def _apply_until(self, pubsub: ConfigSyncPubSub, deadline: float) -> None:
+        while True:
+            remaining = deadline - self._monotonic()
+            if remaining <= 0:
+                return
+            message = await pubsub.get_message(
+                ignore_subscribe_messages=True, timeout=min(remaining, _POLL_TIMEOUT_SECONDS)
+            )
+            if message is not None:
+                await self._apply(pubsub, message)
 
     @staticmethod
     async def _drain_pending(pubsub: ConfigSyncPubSub) -> None:
         while await pubsub.get_message(ignore_subscribe_messages=True, timeout=0) is not None:
             pass
 
+    @staticmethod
+    async def _ready_messages(pubsub: ConfigSyncPubSub) -> AsyncIterator[object]:
+        while (message := await pubsub.get_message(ignore_subscribe_messages=True, timeout=0)) is not None:
+            yield message
+
+    async def _apply(self, pubsub: ConfigSyncPubSub, message: object) -> None:
+        apply: Final = self._apply_model_ids
+        if apply is None:
+            return
+        ready: Final = (message, *[ready_message async for ready_message in self._ready_messages(pubsub)])
+        model_ids: Final = _published_model_ids(ready)
+        if not model_ids:
+            return
+        try:
+            await apply(model_ids)
+        except Exception as e:  # noqa: BLE001  # the debounced full resync retries what a targeted apply could not
+            verbose_proxy_logger.warning("config sync targeted apply of %s failed: %s", model_ids, e)
+
     async def _run_resync_callbacks(self) -> None:
         for callback in self._resync_callbacks:
             try:

litellm/proxy/common_utils/model_listing_utils.py

@@ -219,7 +219,7 @@ def caller_alias_maps(
 
 def alias_map(aliases: object) -> Mapping[str, str]:
     try:
-        entries: Final = _ALIAS_ENTRIES.validate_python(aliases, strict=True)
+        entries: Final = _ALIAS_ENTRIES.validate_python(aliases)
     except ValidationError:
         return _NO_ALIASES
     return MappingProxyType(
@@ -287,6 +287,15 @@ def _dispatch(model_id: str, layers: Sequence[_AliasLayer]) -> tuple[str, bool]:
     return reduce(_through_layer, layers, (model_id, False))
 
 
+def dispatched_model(model_id: str, aliases: CallerAliases) -> str:
+    target, _ = _dispatch(model_id, _alias_layers(aliases))
+    return target
+
+
+def requestable_alias_names(aliases: CallerAliases) -> frozenset[str]:
+    return frozenset(chain.from_iterable(layer.aliases for layer in _alias_layers(aliases)))
+
+
 def _lookup_id(name: str, lookup_by_response: Mapping[str, str], lookup_ids: Container[str]) -> str | None:
     if name in lookup_by_response:
         return lookup_by_response[name]

litellm/proxy/common_utils/openai_error_payload.py

@@ -28,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
 
 _AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN})
 
+_HTTP_ERROR_STATUSES: Final = range(status.HTTP_400_BAD_REQUEST, 600)
+
 _ERROR_BODY: Final = TypeAdapter(dict[str, object])
 
 _FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"})
@@ -47,13 +49,26 @@ def attribute_of(value: object, name: str, default: object = None) -> object:
 
 
 def error_status_code(exc: object, default: int) -> int:
-    """The HTTP status an exception carries as ``status_code`` or, the way ``ProxyException``
-    stores it, as a stringified ``code``; ``default`` when it carries neither."""
-    carried: Final = attribute_of(exc, "status_code")
-    if isinstance(carried, int) and not isinstance(carried, bool):
+    """The 4xx or 5xx HTTP status an exception carries as ``status_code`` (an int or a decimal string) or,
+    the way ``ProxyException`` stores it, as a stringified ``code``; ``default`` when it carries neither."""
+    carried: Final = _http_error_status(attribute_of(exc, "status_code"))
+    if carried is not None:
         return carried
     stringified: Final = attribute_of(exc, "code")
-    return int(stringified) if isinstance(stringified, str) and stringified.isdecimal() else default
+    from_code: Final = _http_error_status(stringified) if isinstance(stringified, str) else None
+    return default if from_code is None else from_code
+
+
+def _http_error_status(value: object) -> int | None:
+    match value:
+        case bool():
+            return None
+        case int() if value in _HTTP_ERROR_STATUSES:
+            return value
+        case str() if len(value) == 3 and value.isascii() and value.isdigit() and int(value) in _HTTP_ERROR_STATUSES:
+            return int(value)
+        case _:
+            return None
 
 
 def openai_error_type(exc: object, status_code: int) -> str:
@@ -105,9 +120,9 @@ def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyExcept
                 code=status.HTTP_503_SERVICE_UNAVAILABLE,
             )
         case HTTPException(status_code=status_code, detail=detail):
-            return _openai_shaped_proxy_exception(exc, str(detail), status_code)
+            return openai_shaped_proxy_exception(exc, str(detail), status_code)
         case _:
-            return _openai_shaped_proxy_exception(
+            return openai_shaped_proxy_exception(
                 exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code)
             )
 
@@ -176,14 +191,19 @@ def _route_failure_error_type(exc: Exception, raised_status: int, answered_statu
     return openai_error_type(exc, answered_status)
 
 
-def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
+def openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
     answered_status: Final = answered_status_code(message, status_code)
-    return ProxyException(
-        message=message,
-        type=_catch_all_error_type(exc, answered_status),
-        param=openai_error_param(exc),
-        code=answered_status,
-    )
+    return _proxy_exception(exc, message, answered_status, _catch_all_error_type(exc, answered_status))
+
+
+def status_typed_proxy_exception(exc: HTTPException) -> ProxyException:
+    message: Final = str(exc.detail)
+    answered_status: Final = answered_status_code(message, exc.status_code)
+    return _proxy_exception(exc, message, answered_status, openai_error_type(exc, answered_status))
+
+
+def _proxy_exception(exc: Exception, message: str, answered_status: int, error_type: str) -> ProxyException:
+    return ProxyException(message=message, type=error_type, param=openai_error_param(exc), code=answered_status)
 
 
 def _catch_all_error_type(exc: Exception, status_code: int) -> str:

litellm/proxy/common_utils/registry_read_through.py

@@ -15,12 +15,13 @@ or the first routed request on this replica would pick a tier it has not loaded.
 import asyncio
 import time
 from collections.abc import Awaitable, Callable, Mapping, Sequence
-from typing import TYPE_CHECKING, Final
+from typing import TYPE_CHECKING, Final, Literal, TypeAlias
 
 from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
 
 if TYPE_CHECKING:
@@ -33,8 +34,14 @@ if TYPE_CHECKING:
     )
 
     from litellm.integrations.custom_guardrail import CustomGuardrail
+    from litellm.repositories.prisma_protocols import TableActions
     from litellm.types.agents import AgentResponse
 
+    _ModelTable: TypeAlias = TableActions[prisma_models.LiteLLM_ProxyModelTable]
+    _ModelRows: TypeAlias = Sequence[prisma_models.LiteLLM_ProxyModelTable]
+
+ModelLoadOutcome: TypeAlias = Literal["loaded", "missing", "no_router", "auto_router"]
+
 READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0
 READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0
 READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20
@@ -123,47 +130,114 @@ async def _resync_model_deployments(model_name: str) -> bool:
         return False
     prisma_client: Final = proxy_server.prisma_client
     assert prisma_client is not None
-    table: Final = ModelRepository(prisma_client).table
-    name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
-    id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
-    rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
+    rows: Final = await _find_rows_by_name_or_id(model_name, ModelRepository(prisma_client).table)
     if not rows:
         return False
-    router: Final = proxy_server.llm_router
-    if router is None:
-        await proxy_server.proxy_config.add_deployment(
-            prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
-        )
-        return proxy_server.llm_router is not None
-    if _holds_an_auto_router(rows):
-        await proxy_server.proxy_config.add_deployment(
-            prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
-        )
-        return _model_is_loaded(model_name)
     async with proxy_server.MODEL_RECONCILE_LOCK:
+        outcome: Final = await _install_model_rows(rows, _always)
+    if outcome in ("no_router", "auto_router"):
+        await _reload_every_model()
+    return outcome == "loaded" or _model_is_loaded(model_name)
+
+
+async def resync_model_deployments_by_ids(model_ids: tuple[str, ...]) -> None:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
+    from litellm.repositories.model_repository import ModelRepository
+
+    if not _db_backed_registries_enabled("models"):
+        return
+    prisma_client: Final = proxy_server.prisma_client
+    assert prisma_client is not None
+    writer_table: Final = ModelRepository(WriterPinnedClient(prisma_client.db)).table
+    async with proxy_server.MODEL_RECONCILE_LOCK:
+        rows: Final = await _find_rows_by_ids(model_ids, writer_table)
+        _ = await _install_model_rows(_without_auto_routers(rows), _names_an_unloaded_credential)
+
+
+async def _find_rows_by_name_or_id(model_name: str, table: "_ModelTable") -> "_ModelRows":
+    name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
+    id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
+    return await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
+
+
+async def _find_rows_by_ids(model_ids: tuple[str, ...], table: "_ModelTable") -> "_ModelRows":
+    from litellm.repositories.chunked_in import find_many_in
+
+    return list(await find_many_in(table, "model_id", model_ids))
+
+
+async def _install_model_rows(
+    rows: "_ModelRows",
+    needs_credentials: Callable[["_ModelRows"], bool],
+) -> ModelLoadOutcome:
+    from litellm.proxy import proxy_server
+
+    router: Final = proxy_server.llm_router
+    if not rows:
+        return "missing"
+    if router is None:
+        return "no_router"
+    if _holds_an_auto_router(rows):
+        return "auto_router"
+    if needs_credentials(rows):
+        prisma_client: Final = proxy_server.prisma_client
+        assert prisma_client is not None
         await proxy_server.proxy_config.get_credentials(prisma_client=prisma_client)
-        proxy_server.proxy_config._add_deployment(db_models=rows)
-        proxy_server.llm_model_list = router.registry().deployments()
+    proxy_server.proxy_config._add_deployment(db_models=rows)
+    proxy_server.llm_model_list = router.registry().deployments()
+    return "loaded"
+
+
+async def _reload_every_model() -> None:
+    from litellm.proxy import proxy_server
+
+    prisma_client: Final = proxy_server.prisma_client
+    assert prisma_client is not None
+    await proxy_server.proxy_config.add_deployment(
+        prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
+    )
+
+
+def _always(rows: "_ModelRows") -> bool:
     return True
 
 
-def _holds_an_auto_router(rows: Sequence["prisma_models.LiteLLM_ProxyModelTable"]) -> bool:
+def _holds_an_auto_router(rows: "_ModelRows") -> bool:
     return any(_names_an_auto_router(row.litellm_params) for row in rows)
 
 
+def _without_auto_routers(rows: "_ModelRows") -> "_ModelRows":
+    return list(row for row in rows if not _names_an_auto_router(row.litellm_params))
+
+
+def _names_an_unloaded_credential(rows: "_ModelRows") -> bool:
+    return any(_names_an_unloaded_credential_param(row.litellm_params) for row in rows)
+
+
 _STORED_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
 
 
-def _names_an_auto_router(stored_litellm_params: object) -> bool:
+def _decrypted_param(stored_litellm_params: object, key: str) -> str | None:
     try:
         params: Final = _STORED_LITELLM_PARAMS.validate_python(stored_litellm_params)
     except ValidationError:
-        return False
-    stored_model: Final = params.get("model")
-    if not isinstance(stored_model, str):
-        return False
-    model: Final = decrypt_value_helper(value=stored_model, key="model", return_original_value=True)
-    return isinstance(model, str) and model.startswith("auto_router/")
+        return None
+    stored_value: Final = params.get(key)
+    if not isinstance(stored_value, str):
+        return None
+    value: Final = decrypt_value_helper(value=stored_value, key=key, return_original_value=True)
+    return value if isinstance(value, str) else None
+
+
+def _names_an_auto_router(stored_litellm_params: object) -> bool:
+    model: Final = _decrypted_param(stored_litellm_params, "model")
+    return model is not None and model.startswith("auto_router/")
+
+
+def _names_an_unloaded_credential_param(stored_litellm_params: object) -> bool:
+    credential_name: Final = _decrypted_param(stored_litellm_params, "litellm_credential_name")
+    return credential_name is not None and CredentialAccessor.find_credential(credential_name) is None
 
 
 async def _resync_guardrails(guardrail_name: str) -> bool:

litellm/proxy/db/db_spend_update_writer.py

@@ -34,6 +34,7 @@ from litellm.constants import (
     DB_SPEND_UPDATE_JOB_NAME,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
 )
+from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line
 from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.proxy._types import (
@@ -652,7 +653,14 @@ class DBSpendUpdateWriter:
                 end_time=end_time,
                 llm_router=llm_router,
             )
-            payload["spend"] = response_cost or 0.0
+            billable_cost: Final = finite_cost(response_cost)
+            if response_cost is not None and billable_cost is None:
+                verbose_proxy_logger.warning(
+                    "Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s",
+                    response_cost,
+                    single_log_line(payload["request_id"]),
+                )
+            payload["spend"] = billable_cost or 0.0
             if isinstance(payload["startTime"], datetime):
                 payload["startTime"] = payload["startTime"].isoformat()
             if isinstance(payload["endTime"], datetime):
@@ -702,7 +710,7 @@ class DBSpendUpdateWriter:
             spawn_logging_task(
                 asyncio.get_running_loop(),
                 self._batch_database_updates(
-                    response_cost=response_cost,
+                    response_cost=billable_cost,
                     user_id=user_id,
                     hashed_token=hashed_token,
                     team_id=team_id,

litellm/proxy/db/db_transaction_queue/redis_update_buffer.py

@@ -8,7 +8,7 @@ import asyncio
 import json
 from collections.abc import Mapping, Sequence
 from datetime import datetime
-from functools import reduce
+from functools import partial, reduce
 from itertools import chain
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
 
@@ -109,6 +109,16 @@ _ValueT = TypeVar("_ValueT")
 _DECODED_JSON: Final = TypeAdapter(object)
 
 
+def _zero_for_non_finite_constant(constant: str) -> float:
+    verbose_proxy_logger.warning(
+        "Redis spend buffer - reading a parked %s spend as 0, the spend it held is lost", constant
+    )
+    return 0.0
+
+
+_load_buffered_transaction: Final = partial(json.loads, parse_constant=_zero_for_non_finite_constant)
+
+
 def _spend_log_json_default(value: object) -> str:
     return value.isoformat() if isinstance(value, datetime) else str(value)
 
@@ -118,7 +128,7 @@ def _encode_spend_log_row(row: SpendLogRow) -> str:
 
 
 def _decode_spend_log_row(encoded: str) -> dict[str, object] | None:
-    decoded: Final = _DECODED_JSON.validate_python(json.loads(encoded))
+    decoded: Final = _DECODED_JSON.validate_python(_load_buffered_transaction(encoded))
     return decoded if isinstance(decoded, dict) else None
 
 
@@ -825,13 +835,13 @@ class RedisUpdateBuffer:
             if slot_result is None:
                 daily_results.append(None)
             else:
-                list_of_daily = [json.loads(t) for t in slot_result]
+                list_of_daily = [_load_buffered_transaction(t) for t in slot_result]
                 aggregated = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(list_of_daily)
                 daily_results.append(aggregated)
 
         window_spend: Final = (
             WindowSpendUpdateQueue.get_aggregated_window_spend_transactions(
-                tuple(json.loads(transaction) for transaction in raw_results[6])
+                tuple(_load_buffered_transaction(transaction) for transaction in raw_results[6])
             )
             if raw_results[6] is not None
             else None
@@ -880,7 +890,7 @@ class RedisUpdateBuffer:
         )
         if list_of_transactions is None:
             return None
-        return [json.loads(transaction) for transaction in list_of_transactions]
+        return [_load_buffered_transaction(transaction) for transaction in list_of_transactions]
 
     async def get_all_daily_spend_update_transactions_from_redis_buffer(
         self,
@@ -1010,9 +1020,9 @@ class RedisUpdateBuffer:
         Parses the list of transactions from Redis
         """
         if isinstance(list_of_transactions, list):
-            return [json.loads(transaction) for transaction in list_of_transactions]
+            return [_load_buffered_transaction(transaction) for transaction in list_of_transactions]
         else:
-            return [json.loads(list_of_transactions)]
+            return [_load_buffered_transaction(list_of_transactions)]
 
     @staticmethod
     def _combine_list_of_transactions(

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -22,6 +22,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX
+from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
 from litellm.proxy.common_utils.path_utils import is_within, safe_join
 from litellm.proxy.guardrails.content_filter_data import CATEGORIES_DIR, DATA_ROOTS, category_dirs, find_category_file
 from litellm.proxy.guardrails.guardrail_hooks.custom_code.bounded_execution import (
@@ -2533,7 +2534,7 @@ async def apply_guardrail(
                 e = transformed_exception
         except Exception:
             verbose_proxy_logger.exception("apply_guardrail: post_call_failure_hook failed")
-        raise handle_exception_on_proxy(e)
+        raise handle_exception_on_proxy(status_typed_proxy_exception(e) if isinstance(e, HTTPException) else e)
 
     # Success logging outside except so a hook error never triggers failure handlers.
     response = await _emit_guardrail_success_logs(

litellm/proxy/health_check.py

@@ -32,7 +32,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import (
     classify_strategy_router_model,
     strategy_router_dependencies,
 )
-from litellm.router_utils.common_utils import is_model_capability_field
+from litellm.router_utils.common_utils import deployment_model_id_pin, is_model_capability_field
 from litellm.types.utils import secret_bearing_wif_litellm_params, server_owned_wif_litellm_params
 
 # Provider routing and server-owned federation or OAuth token-exchange fields. Allowed for proxy
@@ -855,6 +855,8 @@ def update_litellm_params_for_health_check(
       `deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model`
       in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept,
       and `modalities` set there without "audio" turns the audio fields off
+    - sends a SageMaker deployment's `X-Amzn-SageMaker-Inference-Component` extra header as `model_id` when the
+      deployment sets none, so legacy `sagemaker/` probes reach that component the way routed requests do
     """
     incoming_model: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model")
     base_model_owner: Final = deployment_model or (incoming_model if isinstance(incoming_model, str) else None)
@@ -935,9 +937,11 @@ def update_litellm_params_for_health_check(
     audio_output_params: Final = _health_check_audio_output_params(
         _MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner
     )
+    model_id_pin: Final = deployment_model_id_pin(probe_params)
+    model_id_params: Final = {} if model_id_pin is None else {"model_id": model_id_pin}
     return {
         key: value
-        for key, value in {**audio_output_params, **probe_params}.items()
+        for key, value in {**audio_output_params, **model_id_params, **probe_params}.items()
         if not is_model_capability_field(key)
     }
 

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -7,13 +7,15 @@ import secrets
 import time
 import traceback
 from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
+from itertools import chain, groupby
 from types import MappingProxyType
 from typing import Final, Literal, TypedDict
 
 import fastapi
 from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
-from pydantic import TypeAdapter
+from pydantic import BaseModel, ConfigDict, TypeAdapter
 from typing_extensions import ReadOnly
 
 import litellm
@@ -31,7 +33,9 @@ from litellm.proxy._types import (
     CallInfo,
     EnterpriseLicenseData,
     Litellm_EntityType,
+    LiteLLM_UserTable,
     LitellmUserRoles,
+    ModelAccessDeniedProxyException,
     ProxyException,
     SpecialModelNames,
     UserAPIKeyAuth,
@@ -39,13 +43,28 @@ from litellm.proxy._types import (
 )
 from litellm.proxy.auth.auth_checks import (
     _resolve_key_models_for_auth_check,  # pyright: ignore[reportPrivateUsage]  # the auth layer's sentinel resolution, reused so /health scopes exactly like a request
+    agent_model_grants,
+    can_agent_call_model,
+    can_key_call_resolved_model,
+    can_user_call_model,
+    get_user_object,
+    key_model_aliases_for_auth_check,
+    model_matches_any_wildcard_pattern_in_list,
+    team_model_aliases_for_auth_check,
+)
+from litellm.proxy.auth.auth_exception_handler import (
+    _as_proxy_exception,  # pyright: ignore[reportPrivateUsage]  # one auth-failure mapping, shared with user_api_key_auth
 )
 from litellm.proxy.auth.auth_utils import (
     _BANNED_REQUEST_BODY_PARAMS,  # pyright: ignore[reportPrivateUsage]  # one canonical list, shared with the request-body check
 )
-from litellm.proxy.auth.model_checks import get_key_models
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils import http_parsing_utils
+from litellm.proxy.common_utils.model_listing_utils import (
+    CallerAliases,
+    dispatched_model,
+    requestable_alias_names,
+)
 from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
 from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
@@ -79,6 +98,7 @@ from litellm.router_utils.clientside_credential_handler import (
     _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE,  # pyright: ignore[reportPrivateUsage]  # one canonical list, shared with the router path
     clientside_credential_keys,
 )
+from litellm.router_utils.common_utils import resolve_model_group_alias
 from litellm.secret_managers.main import get_secret_bool
 from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields
 from litellm.types.router import reject_server_owned_wif_fields
@@ -253,6 +273,28 @@ def get_callback_identifier(callback):
 
 router: Final = APIRouter()
 _OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
+_OBJECTS: Final = TypeAdapter(tuple[object, ...])
+_DEPLOYMENTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
+
+
+class _ModelGrants(BaseModel):
+    model_config = ConfigDict(frozen=True, from_attributes=True)
+
+    models: tuple[object, ...] = ()
+
+
+class _ModelInfoAccessGroups(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    access_groups: tuple[object, ...]
+
+
+class _DeploymentAccessGroups(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model_info: _ModelInfoAccessGroups
+
+
 services = (
     Literal[
         "slack_budget_alerts",
@@ -1045,42 +1087,339 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict:
     return out
 
 
-def _health_accessible_model_names(
-    user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
-) -> frozenset[str] | None:
-    """Model names the caller may health-check, or None when the key is unrestricted."""
+def _key_models_unrestricted(user_api_key_dict: UserAPIKeyAuth) -> bool:
     granted_models: Final = _resolve_key_models_for_auth_check(user_api_key_dict)
-    if not granted_models or SpecialModelNames.all_proxy_models.value in granted_models:
+    return not granted_models or SpecialModelNames.all_proxy_models.value in granted_models
+
+
+def _own_team_names(deployment: object, team_id: str | None) -> tuple[str, str] | None:
+    match deployment:
+        case {
+            "model_name": str(internal_name),
+            "model_info": {"team_id": str(owner_team_id), "team_public_model_name": str(public_name)},
+        } if owner_team_id == team_id:
+            return (internal_name, public_name)
+        case _:
+            return None
+
+
+def _names_caller_may_request(deployment: object, team_id: str | None) -> frozenset[str]:
+    team_names: Final = _own_team_names(deployment, team_id)
+    if team_names is not None:
+        return frozenset(team_names)
+    match deployment:
+        case {"model_name": str(name)}:
+            return frozenset((name,))
+        case _:
+            return frozenset()
+
+
+def _public_names_by_internal_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, str]:
+    team_names: Final = (_own_team_names(deployment, team_id) for deployment in llm_model_list)
+    return MappingProxyType(dict(names for names in team_names if names is not None))
+
+
+async def _key_owner(user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_UserTable | None:
+    if user_api_key_dict.team_id is not None or user_api_key_dict.user_id is None:
         return None
+    from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
+
+    try:
+        return await get_user_object(
+            user_id=user_api_key_dict.user_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            user_id_upsert=False,
+            parent_otel_span=user_api_key_dict.parent_otel_span,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+    except Exception:  # noqa: BLE001  # auth reads an unloadable user as no user limit, so /health does too
+        return None
+
+
+async def _key_may_call_model(
+    model_name: str,
+    user_api_key_dict: UserAPIKeyAuth,
+    owner: LiteLLM_UserTable | None,
+    llm_router: Router | None,
+) -> bool:
+    from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
+
+    await asyncio.sleep(0)
+    try:
+        await can_key_call_resolved_model(
+            model=model_name, llm_model_list=None, valid_token=user_api_key_dict, llm_router=llm_router
+        )
+        await can_agent_call_model(
+            model=model_name,
+            valid_token=user_api_key_dict,
+            llm_router=llm_router,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+        await can_user_call_model(
+            model=model_name,
+            llm_router=llm_router,
+            user_object=owner,
+            key_model_aliases=key_model_aliases_for_auth_check(user_api_key_dict),
+        )
+    except ModelAccessDeniedProxyException:
+        return False
+    except ProxyException:
+        raise
+    except Exception as failure:
+        if isinstance(failure, HTTPException) and failure.status_code == status.HTTP_403_FORBIDDEN:
+            return False
+        raise _as_proxy_exception(failure) from failure
+    return True
+
+
+async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth) -> CallerAliases:
+    from litellm.proxy.proxy_server import prisma_client, proxy_config, proxy_logging_obj
+
+    router_settings: Final = await proxy_config.get_hierarchical_router_settings(
+        user_api_key_dict, prisma_client, proxy_logging_obj
+    )
+    return CallerAliases(
+        team_model_aliases_for_auth_check(user_api_key_dict),
+        key_model_aliases_for_auth_check(user_api_key_dict),
+        router_settings.get("model_group_alias") if router_settings is not None else None,
+    )
+
+
+def _model_auth_checks(model_name: str, aliases: CallerAliases) -> str:
+    return resolve_model_group_alias(aliases.router, model_name) or model_name
+
+
+def _deployment_id(deployment: object) -> str | None:
+    match deployment:
+        case {"model_info": {"id": str(model_id)}}:
+            return model_id
+        case _:
+            return None
+
+
+def _granted_names(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]:
+    grants: Final = (
+        *_ModelGrants.model_validate(user_api_key_dict).models,
+        *_OBJECTS.validate_python(user_api_key_dict.team_models),
+    )
+    return frozenset(name for name in grants if isinstance(name, str))
+
+
+async def _names_any_limit_grants(user_api_key_dict: UserAPIKeyAuth, owner: LiteLLM_UserTable | None) -> frozenset[str]:
+    owner_grants: Final = _ModelGrants.model_validate(owner).models if owner is not None else ()
+    return (
+        _granted_names(user_api_key_dict)
+        | frozenset(name for name in owner_grants if isinstance(name, str))
+        | await agent_model_grants(user_api_key_dict)
+    )
+
+
+def _alias_names_caller_may_request(aliases: CallerAliases, llm_router: Router | None) -> frozenset[str]:
+    router_aliases: Final = frozenset(llm_router.registry().aliases()) if llm_router is not None else frozenset[str]()
+    return requestable_alias_names(aliases) | router_aliases
+
+
+def _served_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str:
+    dispatched: Final = dispatched_model(model_name, aliases)
+    return llm_router.registry().resolve(dispatched) if llm_router is not None else dispatched
+
+
+def _name_id_pairs(deployment: object, team_id: str | None) -> tuple[tuple[str, str], ...]:
+    model_id: Final = _deployment_id(deployment)
+    if model_id is None:
+        return ()
+    return tuple((name, model_id) for name in _names_caller_may_request(deployment, team_id) | {model_id})
+
+
+def _ids_by_requested_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, frozenset[str]]:
+    pairs: Final = sorted(chain.from_iterable(_name_id_pairs(deployment, team_id) for deployment in llm_model_list))
+    return MappingProxyType(
+        {name: frozenset(model_id for _, model_id in group) for name, group in groupby(pairs, key=lambda pair: pair[0])}
+    )
+
+
+def _ids_of(deployments: Iterable[object]) -> frozenset[str]:
+    return frozenset(model_id for model_id in map(_deployment_id, deployments) if model_id is not None)
+
+
+def _access_groups_of(deployment: object) -> frozenset[str]:
+    match deployment:
+        case {"model_info": {"access_groups": list()}}:
+            access_groups: Final = _DeploymentAccessGroups.model_validate(deployment).model_info.access_groups
+            return frozenset(access_group for access_group in access_groups if isinstance(access_group, str))
+        case _:
+            return frozenset()
+
+
+@dataclass(frozen=True, slots=True)
+class _AccessGroupGrants:
+    granted_names: frozenset[str]
+    ids_in_granted_groups: frozenset[str]
+
+    def requested_name(self, name: str, public_names: Mapping[str, str], admitted_names: frozenset[str]) -> str:
+        public_name: Final = public_names.get(name)
+        if public_name is None or public_name not in admitted_names or self._grants(name):
+            return name
+        return public_name
+
+    def _grants(self, name: str) -> bool:
+        return name in self.granted_names or model_matches_any_wildcard_pattern_in_list(name, tuple(self.granted_names))
+
+    def narrow(self, serving_ids: frozenset[str], served_names: frozenset[str]) -> frozenset[str]:
+        in_granted_groups: Final = serving_ids & self.ids_in_granted_groups
+        granted_directly: Final = not self.granted_names.isdisjoint(served_names)
+        return serving_ids if granted_directly or not in_granted_groups else in_granted_groups
+
+
+def _access_group_grants(llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth) -> _AccessGroupGrants:
+    granted: Final = _granted_names(user_api_key_dict)
+    in_granted_groups: Final = (
+        deployment for deployment in llm_model_list if not granted.isdisjoint(_access_groups_of(deployment))
+    )
+    return _AccessGroupGrants(granted, _ids_of(in_granted_groups))
+
+
+def _ids_serving_group(
+    group: str,
+    ids_by_name: Mapping[str, frozenset[str]],
+    user_api_key_dict: UserAPIKeyAuth,
+    llm_router: Router | None,
+    access_group_grants: _AccessGroupGrants,
+) -> frozenset[str]:
+    named: Final = ids_by_name.get(group)
+    if named is not None:
+        return access_group_grants.narrow(named, frozenset((group,)))
     if llm_router is None:
-        return frozenset(granted_models)
-    registry: Final = llm_router.registry()
+        return frozenset()
+    team_id: Final = user_api_key_dict.team_id
+    wildcard_served: Final = llm_router.registry().deployments(group, source="wildcard", team_id=team_id)
+    served_names: Final = frozenset((group,)).union(
+        *(_names_caller_may_request(deployment, team_id) for deployment in wildcard_served)
+    )
+    return access_group_grants.narrow(_ids_of(wildcard_served), served_names)
+
+
+async def _names_key_may_request(
+    names: Iterable[str],
+    aliases: CallerAliases,
+    user_api_key_dict: UserAPIKeyAuth,
+    owner: LiteLLM_UserTable | None,
+    llm_router: Router | None,
+) -> frozenset[str]:
     return frozenset(
-        get_key_models(
-            user_api_key_dict=user_api_key_dict,
-            proxy_model_list=registry.names(team_id=user_api_key_dict.team_id),
-            model_access_groups=registry.access_groups(),
+        [
+            name
+            for name in names
+            if await _key_may_call_model(_model_auth_checks(name, aliases), user_api_key_dict, owner, llm_router)
+        ]
+    )
+
+
+def _ids_serving(
+    groups: Iterable[str],
+    ids_by_name: Mapping[str, frozenset[str]],
+    user_api_key_dict: UserAPIKeyAuth,
+    llm_router: Router | None,
+    access_group_grants: _AccessGroupGrants,
+) -> frozenset[str]:
+    return frozenset[str]().union(
+        *(
+            _ids_serving_group(group, ids_by_name, user_api_key_dict, llm_router, access_group_grants)
+            for group in groups
         )
     )
 
 
-def _caller_may_probe_deployment(
-    deployment: Mapping[str, object],
-    allowed_models: frozenset[str] | None,
+def _shadowed_ids(
+    llm_model_list: Sequence[object], admitted_names: frozenset[str], reached_ids: frozenset[str], team_id: str | None
+) -> frozenset[str]:
+    ids_of_admitted_names: Final = (
+        _deployment_id(deployment)
+        for deployment in llm_model_list
+        if not admitted_names.isdisjoint(_names_caller_may_request(deployment, team_id))
+    )
+    return frozenset(model_id for model_id in ids_of_admitted_names if model_id is not None) - reached_ids
+
+
+@dataclass(frozen=True, slots=True)
+class _ReachableDeployments:
+    model_ids: frozenset[str]
+    groups: frozenset[str]
+
+    def include(self, deployment: object, team_id: str | None) -> bool:
+        model_id: Final = _deployment_id(deployment)
+        if model_id is None:
+            return not self.groups.isdisjoint(_names_caller_may_request(deployment, team_id))
+        return model_id in self.model_ids
+
+
+_NOTHING_REACHABLE: Final = _ReachableDeployments(frozenset(), frozenset())
+
+
+async def _deployments_key_may_reach(
+    llm_model_list: Sequence[object],
+    aliases: CallerAliases,
+    user_api_key_dict: UserAPIKeyAuth,
+    owner: LiteLLM_UserTable | None,
     llm_router: Router | None,
+) -> _ReachableDeployments:
+    candidate_names: Final = (await _names_any_limit_grants(user_api_key_dict, owner)).union(
+        _alias_names_caller_may_request(aliases, llm_router),
+        *(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list),
+    )
+    admitted_names: Final = await _names_key_may_request(candidate_names, aliases, user_api_key_dict, owner, llm_router)
+    access_group_grants: Final = _access_group_grants(llm_model_list, user_api_key_dict)
+    public_names: Final = _public_names_by_internal_name(llm_model_list, user_api_key_dict.team_id)
+    groups: Final = frozenset(
+        _served_group(access_group_grants.requested_name(name, public_names, admitted_names), aliases, llm_router)
+        for name in admitted_names
+    )
+    ids_by_name: Final = _ids_by_requested_name(llm_model_list, user_api_key_dict.team_id)
+    reached_ids: Final = _ids_serving(groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants)
+    admitted_ids: Final = await _names_key_may_request(
+        _shadowed_ids(llm_model_list, admitted_names, reached_ids, user_api_key_dict.team_id),
+        aliases,
+        user_api_key_dict,
+        owner,
+        llm_router,
+    )
+    id_groups: Final = frozenset(_served_group(model_id, aliases, llm_router) for model_id in admitted_ids)
+    return _ReachableDeployments(
+        reached_ids | _ids_serving(id_groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants), groups
+    )
+
+
+def _model_param_target(
+    model: str,
+    aliases: CallerAliases,
+    llm_model_list: Sequence[Mapping[str, object]],
     team_id: str | None,
-    caller_is_admin: bool,
+    llm_router: Router | None,
+) -> str:
+    group: Final = _served_group(model, aliases, llm_router)
+    if llm_router is None or deployments_targeted_by_name(llm_model_list, group, team_id):
+        return group
+    patterns: Final = frozenset(
+        deployment.get("model_name")
+        for deployment in llm_router.registry().deployments(group, source="wildcard", team_id=team_id)
+    )
+    match tuple(patterns):
+        case (str(pattern),):
+            return pattern
+        case _:
+            return group
+
+
+def _caller_may_probe_deployment(
+    deployment: Mapping[str, object], reachable: _ReachableDeployments, team_id: str | None, caller_is_admin: bool
 ) -> bool:
     """Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included."""
     if not caller_is_admin and not deployment_usable_by_team(deployment, team_id):
         return False
-    if allowed_models is None:
-        return True
-    if llm_router is None:
-        return deployment.get("model_name") in allowed_models
-    model: Final = dict(deployment)
-    registry: Final = llm_router.registry()
-    return any(registry.includes(model, model_name=name, team_id=team_id) for name in allowed_models)
+    return reachable.include(deployment, team_id)
 
 
 def _resolve_targeted_model_ids(
@@ -1137,6 +1476,7 @@ def _filter_health_check_results_by_model_ids(results: dict, allowed_model_ids:
 async def _perform_health_check_and_save(
     model_list,
     target_model,
+    recorded_model_name: str | None,
     cli_model,
     details,
     prisma_client,
@@ -1159,8 +1499,7 @@ async def _perform_health_check_and_save(
 
     # Optionally save health check result to database (non-blocking)
     if prisma_client is not None:
-        # For CLI model, use cli_model name; for router models, use target_model
-        model_name_for_db: Final = cli_model if cli_model is not None else target_model
+        model_name_for_db: Final = cli_model if cli_model is not None else recorded_model_name
         if model_name_for_db is not None:
             asyncio.create_task(
                 _save_health_check_to_db(
@@ -1290,6 +1629,7 @@ async def health_endpoint(
                 cli_result: Final = await _perform_health_check_and_save(
                     model_list=[],
                     target_model=None,
+                    recorded_model_name=None,
                     cli_model=user_model,
                     details=health_check_details,
                     prisma_client=prisma_client,
@@ -1304,15 +1644,39 @@ async def health_endpoint(
                 status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                 detail={"error": "Model list not initialized"},
             )
-        allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router)
-        restrict_to_allowed_models: Final = not is_admin or allowed_models is not None
+        restrict_to_allowed_models: Final = not is_admin or not _key_models_unrestricted(user_api_key_dict)
+        aliases: Final = await _caller_aliases(user_api_key_dict) if restrict_to_allowed_models else None
+        reachable: Final = (
+            await _deployments_key_may_reach(
+                _OBJECTS.validate_python(llm_model_list),
+                aliases,
+                user_api_key_dict,
+                await _key_owner(user_api_key_dict),
+                llm_router,
+            )
+            if aliases is not None
+            else _NOTHING_REACHABLE
+        )
+        requested_model: Final = (
+            _model_param_target(
+                model,
+                aliases,
+                _DEPLOYMENTS.validate_python(llm_model_list),
+                user_api_key_dict.team_id,
+                llm_router,
+            )
+            if model is not None and aliases is not None
+            else model
+        )
         _llm_model_list: Final = [
             m
             for m in copy.deepcopy(llm_model_list)
             if not restrict_to_allowed_models
-            or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin)
+            or _caller_may_probe_deployment(m, reachable, user_api_key_dict.team_id, is_admin)
         ]
-        targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id)
+        targeted_ids: Final = _resolve_targeted_model_ids(
+            _llm_model_list, requested_model, model_id, user_api_key_dict.team_id
+        )
         if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids:
             raise HTTPException(
                 status_code=status.HTTP_403_FORBIDDEN,
@@ -1370,7 +1734,8 @@ async def health_endpoint(
         else:
             router_result: Final = await _perform_health_check_and_save(
                 model_list=_llm_model_list,
-                target_model=target_model,
+                target_model=target_model if model_id else requested_model,
+                recorded_model_name=target_model,
                 cli_model=None,
                 details=health_check_details,
                 prisma_client=prisma_client,

litellm/proxy/hooks/proxy_track_cost_callback.py

@@ -16,6 +16,7 @@ from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLE
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.core_helpers import (
     budget_reservation_from_metadata,
+    finite_cost,
     get_litellm_metadata_from_kwargs,
     get_metadata_variable_name_from_kwargs,
     get_parent_otel_span_from_kwargs,
@@ -703,7 +704,7 @@ class _ProxyDBLogger(CustomLogger):
             raw_cost: Final = _str_keyed_fields(
                 sl_object if sl_object is not None else kwargs  # pyright: ignore[reportUnknownArgumentType]  # untyped success-callback payload
             ).get("response_cost")
-            priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None
+            priced_cost: Final = finite_cost(raw_cost)
             response_cost: Final = (
                 await apply_search_free_tier(
                     kwargs=kwargs,  # pyright: ignore[reportUnknownArgumentType]  # untyped success-callback payload

litellm/proxy/litellm_pre_call_utils.py

@@ -3188,9 +3188,9 @@ def _fallback_lists(
 ) -> tuple[Sequence[object], ...]:
     """Every fallback list the router can route this request through: the three
     router-level lists, the same three keys from the request body (which replace
-    the router's when present, so the union is the safe superset), and the
-    key/team ``router_settings.fallbacks`` the proxy prefers over router-level
-    fallbacks in ``_configured_fallbacks``."""
+    the router's when present, so the union is the safe superset), and the same
+    three keys from key/team ``router_settings``, which the proxy forwards into
+    the request after this discovery runs."""
     lists: Final[list[Sequence[object]]] = []  # mutable-ok: accumulates each fallback list found
     configured: Final = llm_router.settings()
     for router_list in (
@@ -3205,11 +3205,12 @@ def _fallback_lists(
         if isinstance(entries, list):
             lists.append(cast("Sequence[object]", entries))  # cast-ok: request fallback lists hold mixed entry shapes
     for settings in (router_settings, team_router_settings):
-        key_fallbacks: object = settings.get("fallbacks") if settings is not None else None
-        if isinstance(key_fallbacks, list):
-            lists.append(
-                cast("Sequence[object]", key_fallbacks)  # cast-ok: settings fallback lists hold mixed entry shapes
-            )
+        for key in _REQUEST_FALLBACK_KEYS:
+            key_fallbacks: object = settings.get(key) if settings is not None else None
+            if isinstance(key_fallbacks, list):
+                lists.append(
+                    cast("Sequence[object]", key_fallbacks)  # cast-ok: settings fallback lists hold mixed entry shapes
+                )
     return tuple(lists)
 
 

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -12,7 +12,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
 
 from litellm import constants
 from litellm._logging import verbose_proxy_logger
-from litellm.constants import PTU_SENTINEL_API_KEY
+from litellm.constants import PTU_SENTINEL_API_KEY, SUBSCRIPTION_BACKED_PROVIDERS
 from litellm.proxy._types import CommonProxyErrors
 from litellm.proxy.spend_tracking.key_metadata_recovery import (
     attach_user_details,
@@ -295,6 +295,12 @@ def _entity_metadata(
     return stored if stored is not None else {}
 
 
+def provider_bucket_metadata(provider: str, base: Mapping[str, object]) -> Mapping[str, object]:
+    if provider in SUBSCRIPTION_BACKED_PROVIDERS:
+        return {**base, "subscription_covered": True}
+    return dict(base)
+
+
 def update_breakdown_metrics(
     breakdown: BreakdownMetrics,
     record: DailySpendRecord,
@@ -395,7 +401,7 @@ def update_breakdown_metrics(
         if provider not in breakdown.providers:
             breakdown.providers[provider] = MetricWithMetadata(
                 metrics=SpendMetrics(),
-                metadata=provider_metadata.get(provider, {}),  # Add any provider-specific metadata here
+                metadata=provider_bucket_metadata(provider, provider_metadata.get(provider, {})),
             )
         breakdown.providers[provider].metrics = update_metrics(breakdown.providers[provider].metrics, record)
 
@@ -795,12 +801,23 @@ def _aggregate_grouping_sets_records_sync(
             grouped_data[date_str] = bucket
         return bucket
 
-    def assign_metric_with_metadata(target: dict[str, MetricWithMetadata], key: str, metrics: SpendMetrics) -> None:
+    def assign_metric_with_metadata(
+        target: dict[str, MetricWithMetadata],
+        key: str,
+        metrics: SpendMetrics,
+        metadata: Mapping[str, object] | None = None,
+    ) -> None:
         existing: Final = target.get(key)
-        if existing is None:
-            target[key] = MetricWithMetadata(metrics=metrics, metadata={})
-        else:
-            existing.metrics = sum_spend_metrics((existing.metrics, metrics))
+        target[key] = (
+            MetricWithMetadata(metrics=metrics, metadata=dict(metadata or {}))
+            if existing is None
+            else existing.model_copy(
+                update={
+                    "metrics": sum_spend_metrics((existing.metrics, metrics)),
+                    "metadata": {**existing.metadata, **(metadata or {})},
+                }
+            )
+        )
 
     def assign_api_key_breakdown(
         target: dict[str, MetricWithMetadata],
@@ -864,7 +881,9 @@ def _aggregate_grouping_sets_records_sync(
             # provider the base build reported.
             provider_metrics = metrics.model_copy(update={"flat_cost": 0.0})
             provider = record.custom_llm_provider or "unknown"
-            assign_metric_with_metadata(breakdown.providers, provider, provider_metrics)
+            assign_metric_with_metadata(
+                breakdown.providers, provider, provider_metrics, provider_bucket_metadata(provider, {})
+            )
         elif level == _GROUP_DATE_PROVIDER_API_KEY:
             if not is_ptu_sentinel:
                 provider = record.custom_llm_provider or "unknown"

litellm/proxy/management_endpoints/fallback_management_endpoints.py

@@ -14,10 +14,12 @@ import json
 from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Annotated, Final, Literal
 
-from pydantic import Field, JsonValue, TypeAdapter
+from pydantic import Field, JsonValue, TypeAdapter, ValidationError
 
+import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.auth.auth_utils import fallback_target_model_name
 from litellm.proxy.auth.model_checks import get_all_fallbacks
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.utils import PrismaClient, invalidate_config_param
@@ -26,6 +28,7 @@ if TYPE_CHECKING:
     from fastapi import APIRouter, Depends, HTTPException, status
 
     from litellm.proxy.proxy_server import ProxyConfig
+    from litellm.router import Router
 else:
     try:
         from fastapi import APIRouter, Depends, HTTPException, status
@@ -41,6 +44,7 @@ from litellm.types.management_endpoints.router_settings_endpoints import (
     FallbackGetResponse,
     FallbackResponse,
 )
+from litellm.types.router import ContentPolicyFallbackRule, ContentPolicyFallbackRules, FallbackTargets
 
 router: Final = APIRouter()
 
@@ -50,12 +54,83 @@ StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_m
 _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback])
 _JSON_RULES: Final = TypeAdapter(list[JsonValue])
 _STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue])
+_OPTIONAL_JSON_RULES: Final[TypeAdapter[Sequence[JsonValue] | None]] = TypeAdapter(Sequence[JsonValue] | None)
+_CONTENT_POLICY_FALLBACK_RULE: Final[TypeAdapter[ContentPolicyFallbackRule]] = TypeAdapter(ContentPolicyFallbackRule)
+_FALLBACK_TARGETS: Final[TypeAdapter[FallbackTargets]] = TypeAdapter(FallbackTargets)
 
 
 def _rule_covers(entry: StoredFallback, model: str) -> bool:
     return isinstance(entry, dict) and model in entry
 
 
+def _layered_over_config_rules(
+    runtime_rules: ContentPolicyFallbackRules, config_rules: Sequence[Mapping[str, JsonValue]]
+) -> Sequence[Mapping[str, object]]:
+    covered: Final = frozenset[str]().union(*runtime_rules)
+    return [*runtime_rules, *(rule for rule in config_rules if covered.isdisjoint(rule))]
+
+
+def _config_file_rules(rules: object) -> Sequence[Mapping[str, JsonValue]]:
+    try:
+        candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or ()
+    except ValidationError:
+        return ()
+    return tuple(rule for rule in candidates if isinstance(rule, Mapping))
+
+
+def _log_ignored_content_policy_rules(rules: object, error: ValidationError) -> None:
+    verbose_proxy_logger.error(
+        "Ignoring invalid content_policy_fallbacks %r from the DB, every other rule still applies. "
+        "Fix it through POST /fallback or /config/update: %s",
+        rules,
+        error,
+    )
+
+
+def _checked_content_policy_rule(rule: JsonValue) -> ContentPolicyFallbackRule | ValidationError:
+    try:
+        return _CONTENT_POLICY_FALLBACK_RULE.validate_python(rule)
+    except ValidationError as invalid_rule:
+        return invalid_rule
+
+
+def _checked_content_policy_rules(
+    rules: object,
+) -> Sequence[tuple[object, ContentPolicyFallbackRule | ValidationError]]:
+    try:
+        candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or ()
+    except ValidationError as not_a_rule_list:
+        return ((rules, not_a_rule_list),)
+    return tuple((candidate, _checked_content_policy_rule(candidate)) for candidate in candidates)
+
+
+def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None:
+    checked_rules: Final = _checked_content_policy_rules(rules)
+    runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError))
+    layered_rules: Final = (
+        _layered_over_config_rules(
+            runtime_rules,
+            _config_file_rules(
+                litellm.content_policy_fallbacks  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # untyped module list
+            ),
+        )
+        or None
+    )
+    for rule, result in checked_rules:
+        if log_skipped_rules and isinstance(result, ValidationError):
+            _log_ignored_content_policy_rules(rule, result)
+    if layered_rules == llm_router.settings().content_policy_fallbacks:  # pyright: ignore[reportUnknownMemberType]  # untyped RouterSettings list
+        return
+    llm_router.apply(ChangeSet(settings={"content_policy_fallbacks": layered_rules}))
+
+
+def _apply_fallback_rules(llm_router: "Router", fallback_key: str, rules: Sequence[StoredFallback]) -> None:
+    if fallback_key == "content_policy_fallbacks":
+        apply_content_policy_fallbacks(llm_router, rules, log_skipped_rules=True)
+        return
+    llm_router.apply(ChangeSet(assign={fallback_key: rules}))
+
+
 async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None:
     await proxy_config.get_config()
 
@@ -83,6 +158,7 @@ def _editable_rules(
 async def _write_rules(
     prisma_client: PrismaClient,
     proxy_config: "ProxyConfig",
+    llm_router: "Router",
     stored_settings: Mapping[str, JsonValue],
     fallback_key: str,
     rules: Sequence[StoredFallback],
@@ -91,8 +167,10 @@ async def _write_rules(
         section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)}
     )
     if proxy_config.router_settings.owned_by_config(fallback_key):
+        llm_router.apply(ChangeSet(assign={fallback_key: rules}))
         return
     await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules})
+    _apply_fallback_rules(llm_router, fallback_key, rules)
 
 
 async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None:
@@ -218,10 +296,7 @@ async def create_fallback(
             # Add new fallback
             existing_fallbacks.append({data.model: data.fallback_models})
 
-        await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, existing_fallbacks)
-
-        # Update the in-memory router configuration
-        llm_router.apply(ChangeSet(assign={fallback_key: existing_fallbacks}))
+        await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, existing_fallbacks)
 
         verbose_proxy_logger.info(
             "Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type
@@ -277,7 +352,10 @@ async def get_fallback(
             )
 
         # Get fallbacks using the existing utility function
-        fallback_models: Final = get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
+        fallback_targets: Final = _FALLBACK_TARGETS.validate_python(
+            get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
+        )
+        fallback_models: Final = [fallback_target_model_name(target) or model for target in fallback_targets]
 
         if not fallback_models:
             raise HTTPException(
@@ -367,10 +445,7 @@ async def delete_fallback(
                 detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"},
             )
 
-        await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, updated_fallbacks)
-
-        # Update the in-memory router configuration
-        llm_router.apply(ChangeSet(assign={fallback_key: updated_fallbacks}))
+        await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, updated_fallbacks)
 
         verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type)
 

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -13,6 +13,7 @@ These are members of a Team on LiteLLM
 """
 
 import asyncio
+import contextlib
 import json
 import traceback
 from collections.abc import Awaitable, Mapping, Sequence
@@ -443,30 +444,25 @@ async def add_new_user_to_default_team(
     teams: Sequence[str | NewUserRequestTeam],
     prisma_client: "PrismaClient",
 ):
-    tasks: Final[list[Awaitable[object]]] = []
-    for team in teams:
-        user_role: Literal["user", "admin"] = "user"
-        max_budget_in_team: float | None = None
-        if isinstance(team, str):
-            team_id = team
-        elif isinstance(team, NewUserRequestTeam):
-            team_id = team.team_id
-            user_role = team.user_role
-            max_budget_in_team = team.max_budget_in_team
-        else:
-            raise ValueError(f"Invalid team type: {type(team)}")
-
-        tasks.append(
-            _add_user_to_team(
+    memberships: Final = tuple(_default_team_membership(team) for team in teams)
+    for membership in memberships:
+        with contextlib.suppress(Exception):
+            await _add_user_to_team(
                 user_id=user_id,
-                team_id=team_id,
+                team_id=membership.team_id,
                 user_email=user_email,
                 user_api_key_dict=user_api_key_dict,
-                max_budget_in_team=max_budget_in_team,
-                user_role=user_role,
+                max_budget_in_team=membership.max_budget_in_team,
+                user_role=membership.user_role,
             )
-        )
-    await asyncio.gather(*tasks, return_exceptions=True)
+
+
+def _default_team_membership(team: str | NewUserRequestTeam) -> NewUserRequestTeam:
+    if isinstance(team, str):
+        return NewUserRequestTeam(team_id=team)
+    if isinstance(team, NewUserRequestTeam):
+        return team
+    raise ValueError(f"Invalid team type: {type(team)}")
 
 
 async def _fetch_user_team_ids(user_id: str, prisma_client: "PrismaClient") -> tuple[str, ...]:

litellm/proxy/management_endpoints/model_access_group_management_endpoints.py

@@ -12,23 +12,30 @@ Endpoints here:
 - DELETE /access_group/{access_group}/budget - Clear an access group's shared budget
 """
 
+import asyncio
+import contextlib
+import itertools
 import json
-from collections.abc import Mapping, Sequence
-from datetime import datetime
+import time
+from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from datetime import datetime, timedelta
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Annotated, Any, Final, Protocol
 
 from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel, ConfigDict, TypeAdapter
 from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.config_sync_pubsub import publish_model_rows_written
 from litellm.proxy.common_utils.user_api_key_cache import (
     UserApiKeyCache,
     model_access_group_cache_key,
     model_access_group_registry_cache_key,
 )
+from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper
 from litellm.proxy.management_endpoints.common_utils import validate_budget_duration
 
 # Clear cache and reload models to pick up the access group changes
@@ -116,6 +123,28 @@ class _ModelTableClient(Protocol):
     async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
 
 
+class _ModelTableBatchActions(Protocol):
+    def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ...
+
+
+class _ModelTableBatch(Protocol):
+    @property
+    def litellm_proxymodeltable(self) -> _ModelTableBatchActions: ...
+
+
+class _ModelTableTxActions(Protocol):
+    async def find_many(self) -> Sequence[_DeploymentRow]: ...
+
+
+class _ModelTableTx(Protocol):
+    @property
+    def litellm_proxymodeltable(self) -> _ModelTableTxActions: ...
+
+    async def query_raw(self, query: str, *args: object) -> object: ...
+
+    def batch_(self) -> contextlib.AbstractAsyncContextManager[_ModelTableBatch]: ...
+
+
 class _BudgetRow(Protocol):
     @property
     def budget_id(self) -> str: ...
@@ -168,7 +197,42 @@ class _ModelAccessGroupBudgetTableClient(Protocol):
 
 
 def _model_table(prisma_client: PrismaClient) -> _ModelTableClient:
-    return ModelRepository(prisma_client).table
+    return ModelRepository(WriterPinnedClient(prisma_client.db)).table
+
+
+_TRY_LOCK_ACCESS_GROUP: Final = (
+    "SELECT pg_try_advisory_xact_lock(hashtext('model_access_group'), hashtext($1)) AS locked"
+)
+_LOCK_MODEL_ROWS: Final = (
+    'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = ANY($1::text[]) '
+    "ORDER BY model_id FOR UPDATE SKIP LOCKED"
+)
+_MEMBERSHIP_TX_TIMEOUT: Final = timedelta(seconds=30)
+_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds()
+_LOCK_RETRY_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8)
+_LOCK_RETRY_LONGEST_DELAY_SECONDS: Final = 1.0
+
+
+class _LockedModelRow(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model_id: str
+
+
+_LOCKED_MODEL_ROWS: Final = TypeAdapter(tuple[_LockedModelRow, ...])
+
+
+class _AccessGroupLock(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    locked: bool
+
+
+_ACCESS_GROUP_LOCK: Final = TypeAdapter(tuple[_AccessGroupLock])
+
+
+def _model_table_tx(prisma_client: PrismaClient) -> contextlib.AbstractAsyncContextManager[_ModelTableTx]:
+    return writer_wrapper(prisma_client.db).tx(timeout=_MEMBERSHIP_TX_TIMEOUT)
 
 
 def _model_access_group_budget_table(prisma_client: PrismaClient) -> _ModelAccessGroupBudgetTableClient:
@@ -379,11 +443,10 @@ async def _tag_deployment_with_access_group(
     """Write `access_group` into one deployment's model_info; returns the
     (model_id, updated model_info) pair when a write happened, None when the
     deployment already carried the group."""
-    updated_model_info, was_modified = add_access_group_to_deployment(
-        model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
-        access_group=access_group,
+    updated_model_info: Final = _model_info_with_access_group(
+        model_id=model_id, model_info=model_info, access_group=access_group
     )
-    if not was_modified:
+    if updated_model_info is None:
         return None
     await _model_table(prisma_client).update(
         where={"model_id": model_id},
@@ -393,6 +456,141 @@ async def _tag_deployment_with_access_group(
     return (model_id, updated_model_info)
 
 
+def _model_info_with_access_group(model_id: str, model_info: object, access_group: str) -> Mapping[str, object] | None:
+    updated_model_info, was_modified = add_access_group_to_deployment(
+        model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
+        access_group=access_group,
+    )
+    return updated_model_info if was_modified else None
+
+
+def _model_info_without_access_group(
+    model_id: str, model_info: object, access_group: str
+) -> Mapping[str, object] | None:
+    updated_model_info, was_modified = remove_access_group_from_deployment(
+        model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
+        access_group=access_group,
+    )
+    return updated_model_info if was_modified else None
+
+
+def _membership_rewrite(deployment: _DeploymentRow, selected: bool, access_group: str) -> Mapping[str, object] | None:
+    if selected:
+        return _model_info_with_access_group(
+            model_id=deployment.model_id, model_info=deployment.model_info, access_group=access_group
+        )
+    return _model_info_without_access_group(
+        model_id=deployment.model_id, model_info=deployment.model_info, access_group=access_group
+    )
+
+
+def _membership_rewrites(
+    deployments: Sequence[_DeploymentRow], selected_model_ids: frozenset[str], access_group: str
+) -> tuple[tuple[str, Mapping[str, object]], ...]:
+    rewrites: Final = (
+        (
+            deployment.model_id,
+            _membership_rewrite(
+                deployment=deployment,
+                selected=deployment.model_id in selected_model_ids,
+                access_group=access_group,
+            ),
+        )
+        for deployment in deployments
+    )
+    return tuple((model_id, model_info) for model_id, model_info in rewrites if model_info is not None)
+
+
+def _carries_access_group(model_info: object, access_group: str) -> bool:
+    access_groups: Final = (model_info_as_mapping(model_info) or {}).get("access_groups")
+    return isinstance(access_groups, list) and access_group in access_groups
+
+
+def _model_ids_to_lock(
+    deployments: Sequence[_DeploymentRow], access_group: str, data: UpdateModelGroupRequest
+) -> frozenset[str]:
+    selected_model_ids: Final = frozenset(data.model_ids or ())
+    selected_model_names: Final = frozenset(() if data.model_ids else data.model_names or ())
+    return frozenset(
+        deployment.model_id
+        for deployment in deployments
+        if deployment.model_id in selected_model_ids
+        or deployment.model_name in selected_model_names
+        or _carries_access_group(deployment.model_info, access_group)
+    )
+
+
+async def _raise_404_unless_the_primary_carries(access_group: str, prisma_client: PrismaClient) -> None:
+    if access_group not in _access_groups_of_deployments(await _model_table(prisma_client).find_many()):
+        raise _http_error(404, f"Access group '{access_group}' not found")
+
+
+async def _rewrite_membership_unless_locked(
+    prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest
+) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None:
+    async with _model_table_tx(prisma_client) as tx:
+        (group_lock,) = _ACCESS_GROUP_LOCK.validate_python(await tx.query_raw(_TRY_LOCK_ACCESS_GROUP, access_group))
+        if not group_lock.locked:
+            return None
+        model_ids_to_lock: Final = _model_ids_to_lock(
+            deployments=await tx.litellm_proxymodeltable.find_many(), access_group=access_group, data=data
+        )
+        locked_rows: Final = _LOCKED_MODEL_ROWS.validate_python(
+            await tx.query_raw(_LOCK_MODEL_ROWS, sorted(model_ids_to_lock))
+        )
+        if frozenset(row.model_id for row in locked_rows) != model_ids_to_lock:
+            return None
+        deployments: Final = tuple(
+            deployment
+            for deployment in await tx.litellm_proxymodeltable.find_many()
+            if deployment.model_id in model_ids_to_lock
+        )
+        if access_group not in _access_groups_of_deployments(deployments):
+            raise HTTPException(
+                status_code=404,
+                detail={"error": f"Access group '{access_group}' not found"},
+            )
+        selected_deployments: Final = _selected_deployments_or_400(deployments=deployments, data=data)
+        rewrites: Final = _membership_rewrites(
+            deployments=deployments,
+            selected_model_ids=frozenset(deployment.model_id for deployment in selected_deployments),
+            access_group=access_group,
+        )
+        if rewrites:
+            async with tx.batch_() as batch:
+                for model_id, model_info in rewrites:
+                    batch.litellm_proxymodeltable.update(
+                        where={"model_id": model_id}, data={"model_info": json.dumps(model_info)}
+                    )
+    return selected_deployments, rewrites
+
+
+def _lock_retry_delays() -> Iterator[float]:
+    return itertools.chain(_LOCK_RETRY_BACKOFF_SECONDS, itertools.repeat(_LOCK_RETRY_LONGEST_DELAY_SECONDS))
+
+
+async def _rewrite_membership_once_unlocked(
+    prisma_client: PrismaClient,
+    access_group: str,
+    data: UpdateModelGroupRequest,
+    retry_budget_seconds: float = _LOCK_RETRY_BUDGET_SECONDS,
+    sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+    clock: Callable[[], float] = time.monotonic,
+) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]:
+    deadline: Final = clock() + retry_budget_seconds
+    for delay in _lock_retry_delays():
+        if (
+            rewritten := await _rewrite_membership_unless_locked(
+                prisma_client=prisma_client, access_group=access_group, data=data
+            )
+        ) is not None:
+            return rewritten
+        if clock() + delay > deadline:
+            break
+        await sleep(delay)
+    raise _http_error(409, f"Another edit holds access group '{access_group}' or one of its models, try again")
+
+
 def _readable_model_info_or_raise(model_id: str, model_info: object) -> Mapping[str, object]:
     """These helpers rewrite the model_info column wholesale, so a present-but-unreadable
     value must refuse loudly rather than be silently replaced with a fresh object; an
@@ -412,11 +610,10 @@ async def _strip_access_group_from_deployment(
     """Remove `access_group` from one deployment's model_info; returns the
     (model_id, updated model_info) pair when a write happened, None when the
     deployment did not carry the group."""
-    updated_model_info, was_modified = remove_access_group_from_deployment(
-        model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
-        access_group=access_group,
+    updated_model_info: Final = _model_info_without_access_group(
+        model_id=model_id, model_info=model_info, access_group=access_group
     )
-    if not was_modified:
+    if updated_model_info is None:
         return None
     await _model_table(prisma_client).update(
         where={"model_id": model_id},
@@ -493,6 +690,35 @@ async def update_specific_deployments_with_access_group(
     return tuple(pair for pair in tagged if pair is not None)
 
 
+def _selected_deployments_or_400(
+    deployments: Sequence[_DeploymentRow], data: UpdateModelGroupRequest
+) -> tuple[_DeploymentRow, ...]:
+    if data.model_ids:
+        known_model_ids: Final = frozenset(deployment.model_id for deployment in deployments)
+        unknown_model_id: Final = next(
+            (model_id for model_id in data.model_ids if model_id not in known_model_ids), None
+        )
+        if unknown_model_id is not None:
+            raise HTTPException(
+                status_code=400,
+                detail={"error": f"Deployment with model_id '{unknown_model_id}' not found in Database."},
+            )
+        selected_model_ids: Final = frozenset(data.model_ids)
+        return tuple(deployment for deployment in deployments if deployment.model_id in selected_model_ids)
+    model_names: Final = data.model_names or ()
+    known_model_names: Final = frozenset(deployment.model_name for deployment in deployments)
+    unknown_model_name: Final = next((name for name in model_names if name not in known_model_names), None)
+    if unknown_model_name is not None:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": f"Can't find model '{unknown_model_name}' in Database. Access group management is only supported for database models."
+            },
+        )
+    selected_model_names: Final = frozenset(model_names)
+    return tuple(deployment for deployment in deployments if deployment.model_name in selected_model_names)
+
+
 async def _find_deployment_or_400(model_id: str, prisma_client: PrismaClient) -> object:
     deployment: Final = await _model_table(prisma_client).find_unique(where={"model_id": model_id})
     if deployment is None:
@@ -536,10 +762,10 @@ async def get_all_access_groups_from_db(
     Returns:
         Dict[str, AccessGroupInfo]: Dictionary mapping access_group name to info
     """
-    # Get all deployments
-    deployments: Final = await ModelRepository(prisma_client).table.find_many()
+    return dict(_access_groups_of_deployments(await ModelRepository(prisma_client).table.find_many()))
 
-    # Build access group map
+
+def _access_groups_of_deployments(deployments: Sequence[_DeploymentRow]) -> Mapping[str, AccessGroupInfo]:
     model_names_by_group: Final[dict[str, list[str]]] = {}
 
     for deployment in deployments:
@@ -835,8 +1061,8 @@ async def update_access_group(
     Update an access group's model names.
     
     This will:
-    1. Remove the access group from all current deployments
-    2. Add the access group to all deployments for the new model_names list
+    1. Add the access group to all deployments for the new model_names list
+    2. Remove the access group from every other deployment
     
     Example:
     ```bash
@@ -858,6 +1084,7 @@ async def update_access_group(
     Raises:
     - HTTPException 400: If any model names don't exist
     - HTTPException 404: If access group not found
+    - HTTPException 409: If another edit holds the group or one of its models for 30 s
     """
     from litellm.proxy.proxy_server import llm_router, prisma_client
 
@@ -879,26 +1106,14 @@ async def update_access_group(
             detail={"error": "Either model_names or model_ids must be provided and non-empty"},
         )
 
-    use_model_ids: Final = has_model_ids
-
-    # Validation: Check if access group exists
     try:
-        access_groups_map: Final = await get_all_access_groups_from_db(prisma_client=prisma_client)
-        if access_group not in access_groups_map:
-            raise HTTPException(
-                status_code=404,
-                detail={"error": f"Access group '{access_group}' not found"},
-            )
+        await _raise_404_unless_the_primary_carries(access_group=access_group, prisma_client=prisma_client)
     except HTTPException:
         raise
     except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail={"error": f"Failed to check access group existence: {e}"},
-        )
+        raise _http_error(500, f"Failed to check access group existence: {e}")
 
-    # Validation: Check if all new models exist (only if using model_names path)
-    if not use_model_ids and has_model_names:
+    if not has_model_ids and has_model_names:
         assert data.model_names is not None
         missing_models: Final = await _missing_models_after_read_through(
             model_names=data.model_names,
@@ -912,43 +1127,19 @@ async def update_access_group(
             )
 
     try:
-        # Step 1: Remove access group from ALL DB deployments (skip config models)
-        all_deployments: Final = await _model_table(prisma_client).find_many()
-
-        stripped: Final = [
-            await _strip_access_group_from_deployment(
-                model_id=deployment.model_id,
-                model_info=deployment.model_info,
-                access_group=access_group,
-                prisma_client=prisma_client,
-            )
-            for deployment in all_deployments
-        ]
-        stripped_pairs: Final = tuple(pair for pair in stripped if pair is not None)
-
-        # Step 2: Add access group using the appropriate method
-        if use_model_ids:
-            assert data.model_ids is not None
-            updated_pairs = await update_specific_deployments_with_access_group(
-                model_ids=data.model_ids,
-                access_group=access_group,
-                prisma_client=prisma_client,
-            )
-        else:
-            assert data.model_names is not None
-            updated_pairs = await update_deployments_with_access_group(
-                model_names=data.model_names,
-                access_group=access_group,
-                prisma_client=prisma_client,
-            )
-        models_updated: Final = len(updated_pairs)
+        selected_deployments, rewrites = await _rewrite_membership_once_unlocked(
+            prisma_client=prisma_client, access_group=access_group, data=data
+        )
+        if rewrites:
+            await publish_model_rows_written(tuple(model_id for model_id, _ in rewrites))
+        models_updated: Final = len(selected_deployments)
 
         # Clear cache and reload models to pick up the access group changes
         live_before_reload: Final = live_model_ids_snapshot()
         reload_outcome: Final = await clear_cache()
         _raise_http_if_reload_degraded_serving(
             before=live_before_reload,
-            written_models=list({**dict(stripped_pairs), **dict(updated_pairs)}.items()),
+            written_models=rewrites,
             access_group=access_group,
             still_desired=reload_outcome.still_desired,
             live_after=reload_outcome.live_after,

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -779,7 +779,11 @@ async def _auto_router_capability_slot(
                 limit=limit,
             )
         yield tables.litellm_proxymodeltable
-    await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable")
+    await publish_config_change(
+        redis_cache=coordination_redis_cache(),
+        object_type="litellm_proxymodeltable",
+        model_ids=() if model_id is None else (model_id,),
+    )
 
 
 ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add"

litellm/proxy/management_endpoints/scim/scim_v2.py

@@ -229,7 +229,11 @@ class UserProvisionerHelpers:
             return None
 
         requested_teams: Final = list(dict.fromkeys(new_user_request.teams or []))
-        new_teams: Final = requested_teams if requested_teams else list(existing_user.teams or [])
+        new_teams: Final = (
+            list(_keep_record_team_order(existing_user.teams or [], requested_teams))
+            if requested_teams
+            else list(existing_user.teams or [])
+        )
 
         if new_user_request.user_id != existing_user.user_id:
             verbose_proxy_logger.info(
@@ -1165,7 +1169,7 @@ async def _handle_team_membership_changes(
     existing_teams_set: Final = set(existing_teams)
     new_teams_set: Final = set(new_teams)
 
-    teams_to_add: Final = new_teams_set - existing_teams_set
+    teams_to_add: Final = tuple(team for team in dict.fromkeys(new_teams) if team not in existing_teams_set)
     teams_to_remove: Final = existing_teams_set - new_teams_set
 
     if teams_to_add or teams_to_remove:
@@ -1948,7 +1952,11 @@ async def update_user(
 
         # SCIM User.groups is readOnly (RFC 7643 4.1.2): IdPs sync membership via /Groups and send
         # no groups or `groups: []` on profile PUTs, so empty means unspecified, not "remove from every team"
-        target_teams: Final = user_data["teams"] or existing_user.teams
+        target_teams: Final = (
+            list(_keep_record_team_order(existing_user.teams, user_data["teams"]))
+            if user_data["teams"]
+            else existing_user.teams
+        )
         await _handle_team_membership_changes(
             user_id=user_id,
             existing_teams=existing_user.teams,
@@ -2241,19 +2249,29 @@ def _handle_name_update(path: str, op_type: str, value: object, scim_metadata: d
             scim_metadata["familyName"] = str(value)
 
 
-def _handle_group_operations(op_type: str, value: object, teams_set: set[str], path: str | None) -> set[str] | None:
-    """Handle group/team membership operations."""
-    group_values = _extract_group_values(value)
-    if not group_values and value is None:
-        group_values = _extract_ids_from_path_filter(path, "groups")
-    if op_type == "replace":
-        return set(group_values)
-    elif op_type == "add":
-        teams_set.update(group_values)
-    elif op_type == "remove":
-        for gid in group_values:
-            teams_set.discard(gid)
-    return None
+def _is_group_path(path: str | None) -> bool:
+    return isinstance(path, str) and path.lower().startswith("groups")
+
+
+def _apply_group_operation(teams: tuple[str, ...], op: SCIMPatchOperation) -> tuple[str, ...]:
+    listed: Final = _extract_group_values(op.value)
+    group_values: Final = listed if listed or op.value is not None else _extract_ids_from_path_filter(op.path, "groups")
+    if op.op == "replace":
+        return tuple(dict.fromkeys(group_values))
+    if op.op == "add":
+        return tuple(dict.fromkeys((*teams, *group_values)))
+    removed: Final = frozenset(group_values)
+    return tuple(team for team in teams if team not in removed)
+
+
+def _keep_record_team_order(record_teams: Sequence[str], target_teams: Sequence[str]) -> tuple[str, ...]:
+    target: Final = frozenset(target_teams)
+    record: Final = tuple(dict.fromkeys(record_teams))
+    known: Final = frozenset(record)
+    return (
+        *(team for team in record if team in target),
+        *(team for team in dict.fromkeys(target_teams) if team not in known),
+    )
 
 
 def _multi_valued_attribute_base(path: str) -> str:
@@ -2520,8 +2538,8 @@ def _handle_generic_metadata(path: str, op_type: str, value: object, metadata: d
 def _apply_patch_ops(
     existing_user: LiteLLM_UserTable,
     patch_ops: SCIMPatchOp,
-) -> tuple[dict[str, object], set[str]]:
-    """Apply patch operations and return update data and final team set."""
+) -> tuple[dict[str, object], tuple[str, ...]]:
+    """Apply patch operations and return update data and the final teams in order."""
     update_data: Final[dict[str, object]] = {}
     stored_metadata: Final[object] = TypeAdapter(object).validate_python(existing_user.model_dump().get("metadata"))
     metadata: Final = _JSON_OBJECT_ADAPTER.validate_python(stored_metadata) if isinstance(stored_metadata, dict) else {}
@@ -2541,9 +2559,6 @@ def _apply_patch_ops(
     if isinstance(enterprise, _InvalidEnterpriseValue):
         _raise_invalid_enterprise_value(enterprise)
 
-    teams_set: Final[set[str]] = set(existing_user.teams or [])
-    replace_team_set: set[str] | None = None
-
     for op in patch_ops.Operations:
         path = op.path.lower() if isinstance(op.path, str) else ""
         value = op.value
@@ -2585,16 +2600,20 @@ def _apply_patch_ops(
             _handle_name_update(path, op_type, value, scim_metadata)
         elif _multi_valued_attribute_base(path) in SCIM_MULTI_VALUED_ATTRIBUTE_METADATA_KEYS:
             _handle_multi_valued_attribute_update(path, op_type, value, metadata)
-        elif path.startswith("groups"):
-            new_replace_set = _handle_group_operations(op_type, value, teams_set, op.path)
-            if new_replace_set is not None:
-                replace_team_set = new_replace_set
-        elif _enterprise_extension_sub_path(path) is not None:
+        elif _is_group_path(path) or _enterprise_extension_sub_path(path) is not None:
             continue
         else:
             _handle_generic_metadata(path, op_type, value, metadata)
 
-    final_team_set: Final = replace_team_set if replace_team_set is not None else teams_set
+    record_teams: Final = tuple(existing_user.teams or [])
+    group_ops: Final = tuple(op for op in patch_ops.Operations if _is_group_path(op.path))
+    last_replace: Final = next((op for op in reversed(group_ops) if op.op == "replace"), None)
+    final_teams: Final = _keep_record_team_order(
+        record_teams,
+        _apply_group_operation(record_teams, last_replace)
+        if last_replace is not None
+        else reduce(_apply_group_operation, group_ops, record_teams),
+    )
     final_metadata: Final = (
         {
             **{key: metadata_value for key, metadata_value in metadata.items() if key != SCIM_ENTERPRISE_METADATA_KEY},
@@ -2605,7 +2624,7 @@ def _apply_patch_ops(
         else {**metadata, "scim_metadata": scim_metadata}
     )
     update_data["metadata"] = final_metadata
-    return update_data, final_team_set
+    return update_data, final_teams
 
 
 def _is_user_not_in_team_error(exc: HTTPException) -> bool:
@@ -2758,7 +2777,7 @@ async def patch_user(
 
         prev_active: Final = _user_scim_active(existing_user)
 
-        update_data, final_team_set = _apply_patch_ops(
+        update_data, final_teams = _apply_patch_ops(
             existing_user=existing_user,
             patch_ops=patch_ops,
         )
@@ -2770,15 +2789,15 @@ async def patch_user(
         await _handle_team_membership_changes(
             user_id=user_id,
             existing_teams=existing_user.teams or [],
-            new_teams=list(final_team_set),
+            new_teams=list(final_teams),
         )
 
-        update_data["teams"] = list(final_team_set)
+        update_data["teams"] = list(final_teams)
 
         admin_group: Final = await _get_scim_admin_group()
         if admin_group is not None:
             update_data["user_role"] = _resolve_scim_user_role(
-                await _scim_groups_from_team_ids(prisma_client, list(final_team_set)),
+                await _scim_groups_from_team_ids(prisma_client, list(final_teams)),
                 admin_group,
                 _default_scim_user_role(),
             )

litellm/proxy/management_endpoints/subscription_accounts.py

@@ -0,0 +1,335 @@
+"""
+Subscription accounts: the operator-entered recurring fee of a provider account whose
+requests are covered by a flat subscription, shown beside metered spend on the Usage page.
+"""
+
+import re
+from collections.abc import Callable, Iterable, Mapping
+from dataclasses import dataclass
+from datetime import date
+from itertools import chain
+from typing import TYPE_CHECKING, Annotated, Final, TypeAlias
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from pydantic import TypeAdapter
+
+from litellm.constants import LITELLM_PROXY_ADMIN_NAME, SUBSCRIPTION_BACKED_PROVIDERS
+from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
+from litellm.llms.subscription_accounts import resolve_subscription_account_id
+from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
+from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth, user_api_key_has_admin_view
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.spend_tracking.subscription_billing_periods import billing_periods_starting_within
+from litellm.proxy.utils import get_prisma_client_or_throw
+from litellm.repositories.subscription_account_repository import (
+    DuplicateSubscriptionAccount,
+    KeepLabel,
+    SetLabel,
+    SubscriptionAccountFeeChanges,
+    SubscriptionAccountFeeFields,
+    SubscriptionAccountRepository,
+    SubscriptionAccountStore,
+)
+from litellm.types.proxy.management_endpoints.subscription_accounts import (
+    BillingPeriod,
+    DeleteSubscriptionAccountRequest,
+    SubscriptionAccountFeeRequest,
+    SubscriptionAccountUsage,
+    SubscriptionUsageResponse,
+    UpdateSubscriptionAccountFeeRequest,
+)
+from litellm.types.router import DeploymentTypedDict
+
+if TYPE_CHECKING:
+    from litellm.router import Router
+
+router: Final = APIRouter(prefix="/subscription_accounts", tags=["subscription accounts"])
+Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]
+
+
+@dataclass(frozen=True, slots=True)
+class SubscriptionDeployment:
+    model_name: str
+    custom_llm_provider: str
+    account_id: str | None
+
+
+AccountIdResolver: TypeAlias = Callable[[str, Mapping[str, object]], str | None]
+_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+
+
+def _declared_provider(litellm_params: Mapping[str, object]) -> str | None:
+    model: Final = litellm_params.get("model")
+    configured_provider: Final = litellm_params.get("custom_llm_provider")
+    return declared_authenticating_provider(
+        model if isinstance(model, str) else None,
+        configured_provider if isinstance(configured_provider, str) else None,
+    )
+
+
+def _model_info(deployment: Mapping[str, object]) -> Mapping[str, object]:
+    return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
+
+
+def _public_model_name(entry: DeploymentTypedDict) -> str:
+    team_public_model_name: Final = _model_info(entry).get("team_public_model_name")
+    return team_public_model_name if isinstance(team_public_model_name, str) else entry["model_name"]
+
+
+def _subscription_deployment(
+    entry: DeploymentTypedDict, account_id_resolver: AccountIdResolver
+) -> SubscriptionDeployment | None:
+    litellm_params: Final = entry["litellm_params"]
+    provider: Final = _declared_provider(litellm_params)
+    if provider is None or provider not in SUBSCRIPTION_BACKED_PROVIDERS:
+        return None
+    return SubscriptionDeployment(
+        model_name=_public_model_name(entry),
+        custom_llm_provider=provider,
+        account_id=account_id_resolver(provider, litellm_params),
+    )
+
+
+def subscription_deployments_from_model_list(
+    model_list: Iterable[DeploymentTypedDict], account_id_resolver: AccountIdResolver
+) -> tuple[SubscriptionDeployment, ...]:
+    candidates: Final = (_subscription_deployment(entry, account_id_resolver) for entry in model_list)
+    return tuple(candidate for candidate in candidates if candidate is not None)
+
+
+def subscription_deployments_of_router(
+    llm_router: "Router | None", account_id_resolver: AccountIdResolver
+) -> tuple[SubscriptionDeployment, ...]:
+    if llm_router is None:
+        return ()
+    registry: Final = llm_router.registry()
+    indexed_names: Final = tuple(registry.index("model_name"))
+    deployments: Final = chain.from_iterable(registry.deployments(name, source="named") or () for name in indexed_names)
+    return subscription_deployments_from_model_list(deployments, account_id_resolver)
+
+
+def get_subscription_deployments() -> tuple[SubscriptionDeployment, ...]:
+    from litellm.proxy.proxy_server import llm_router
+
+    return subscription_deployments_of_router(llm_router, resolve_subscription_account_id)
+
+
+def get_subscription_account_repository() -> SubscriptionAccountStore:
+    return SubscriptionAccountRepository(get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value))
+
+
+Store: TypeAlias = Annotated[SubscriptionAccountStore, Depends(get_subscription_account_repository)]
+Deployments: TypeAlias = Annotated[tuple[SubscriptionDeployment, ...], Depends(get_subscription_deployments)]
+
+
+def _forbidden(user_api_key_dict: UserAPIKeyAuth) -> HTTPException:
+    return HTTPException(
+        status_code=status.HTTP_403_FORBIDDEN,
+        detail={"error": f"{CommonProxyErrors.not_allowed_access.value}, your role={user_api_key_dict.user_role}"},
+    )
+
+
+def _account_usage(
+    custom_llm_provider: str,
+    account_id: str | None,
+    deployments: Iterable[SubscriptionDeployment],
+    fee_row: LiteLLM_SubscriptionAccountTable | None,
+    start_date: date,
+    end_date: date,
+) -> SubscriptionAccountUsage:
+    model_names: Final = tuple(sorted({deployment.model_name for deployment in deployments}))
+    if fee_row is None:
+        return SubscriptionAccountUsage(
+            custom_llm_provider=custom_llm_provider,
+            account_id=account_id,
+            deployments=model_names,
+            fee=None,
+            billing_periods=(),
+            fixed_cost=None,
+        )
+    periods: Final = billing_periods_starting_within(
+        date.fromisoformat(fee_row.billing_period_start), start_date, end_date
+    )
+    return SubscriptionAccountUsage(
+        custom_llm_provider=custom_llm_provider,
+        account_id=account_id,
+        deployments=model_names,
+        fee=fee_row,
+        billing_periods=tuple(BillingPeriod(start=period.start, end=period.end) for period in periods),
+        fixed_cost=len(periods) * fee_row.monthly_fee,
+    )
+
+
+def build_subscription_usage(
+    deployments: Iterable[SubscriptionDeployment],
+    fee_rows: Iterable[LiteLLM_SubscriptionAccountTable],
+    start_date: date,
+    end_date: date,
+) -> SubscriptionUsageResponse:
+    deployment_tuple: Final = tuple(deployments)
+    fees_by_account: Final = {(row.custom_llm_provider, row.account_id): row for row in fee_rows}
+    deployment_keys: Final = {(d.custom_llm_provider, d.account_id) for d in deployment_tuple}
+    account_keys: Final = sorted(deployment_keys | set(fees_by_account), key=lambda key: (key[0], key[1] or "~"))
+    accounts: Final = tuple(
+        _account_usage(
+            provider,
+            account_id,
+            (d for d in deployment_tuple if (d.custom_llm_provider, d.account_id) == (provider, account_id)),
+            fees_by_account.get((provider, account_id)) if account_id is not None else None,
+            start_date,
+            end_date,
+        )
+        for provider, account_id in account_keys
+    )
+    return SubscriptionUsageResponse(
+        start_date=start_date,
+        end_date=end_date,
+        subscription_providers=tuple(sorted(SUBSCRIPTION_BACKED_PROVIDERS)),
+        accounts=accounts,
+    )
+
+
+_DAY_PATTERN: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
+
+
+def _parse_day(value: str) -> date | None:
+    if _DAY_PATTERN.fullmatch(value) is None:
+        return None
+    try:
+        return date.fromisoformat(value)
+    except ValueError:
+        return None
+
+
+def _parse_date(value: str, name: str) -> date:
+    parsed: Final = _parse_day(value)
+    if parsed is None:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail={"error": f"{name} must be YYYY-MM-DD, got {value!r}"}
+        )
+    return parsed
+
+
+@router.get("/usage", response_model=SubscriptionUsageResponse, dependencies=[Depends(user_api_key_auth)])
+async def get_subscription_usage(
+    start_date: Annotated[str, Query(description="First day of the Usage page filter, YYYY-MM-DD")],
+    end_date: Annotated[str, Query(description="Last day of the Usage page filter, YYYY-MM-DD")],
+    user_api_key_dict: Auth,
+    repository: Store,
+    deployments: Deployments,
+) -> SubscriptionUsageResponse:
+    """
+    Subscription-backed accounts behind the proxy's deployments or carrying a stored fee, each with
+    its configured monthly fee and the fixed cost attributed to the date range: the fee once per billing
+    period whose start date falls inside the range, however many deployments share the account.
+    """
+    if not user_api_key_has_admin_view(user_api_key_dict):
+        raise _forbidden(user_api_key_dict)
+    start: Final = _parse_date(start_date, "start_date")
+    end: Final = _parse_date(end_date, "end_date")
+    if end < start:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail={"error": "end_date must not be before start_date"}
+        )
+    return build_subscription_usage(deployments, await repository.list_all(), start, end)
+
+
+def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None:
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise _forbidden(user_api_key_dict)
+
+
+def _actor(user_api_key_dict: UserAPIKeyAuth) -> str:
+    return user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME
+
+
+@router.post("/new", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)])
+async def new_subscription_account(
+    request: SubscriptionAccountFeeRequest,
+    user_api_key_dict: Auth,
+    repository: Store,
+) -> LiteLLM_SubscriptionAccountTable:
+    """Record the monthly fee of one subscription-backed provider account."""
+    _require_proxy_admin(user_api_key_dict)
+    if request.custom_llm_provider not in SUBSCRIPTION_BACKED_PROVIDERS:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={
+                "error": f"{request.custom_llm_provider!r} is not a subscription-backed provider; "
+                f"expected one of {sorted(SUBSCRIPTION_BACKED_PROVIDERS)}"
+            },
+        )
+    created: Final = await repository.create_account(
+        request.custom_llm_provider,
+        request.account_id,
+        SubscriptionAccountFeeFields(
+            monthly_fee=request.monthly_fee,
+            currency=request.currency,
+            billing_period_start=request.billing_period_start.isoformat(),
+            label=request.label,
+        ),
+        _actor(user_api_key_dict),
+    )
+    if isinstance(created, DuplicateSubscriptionAccount):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={
+                "error": f"A fee is already configured for {created.custom_llm_provider} account "
+                f"{created.account_id}; use /subscription_accounts/update to change it"
+            },
+        )
+    return created
+
+
+def _fee_changes(request: UpdateSubscriptionAccountFeeRequest) -> SubscriptionAccountFeeChanges:
+    return SubscriptionAccountFeeChanges(
+        monthly_fee=request.monthly_fee,
+        currency=request.currency,
+        billing_period_start=(
+            request.billing_period_start.isoformat() if request.billing_period_start is not None else None
+        ),
+        label=SetLabel(request.label) if "label" in request.model_fields_set else KeepLabel(),
+    )
+
+
+@router.post("/update", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)])
+async def update_subscription_account(
+    request: UpdateSubscriptionAccountFeeRequest,
+    user_api_key_dict: Auth,
+    repository: Store,
+) -> LiteLLM_SubscriptionAccountTable:
+    """
+    Change the fee, currency, billing period start, or label of a subscription account.
+    Only the fields named in the request are written, so two admins changing different
+    fields at the same time both land.
+    """
+    _require_proxy_admin(user_api_key_dict)
+    updated: Final = await repository.update_fee(
+        request.subscription_account_id, _fee_changes(request), _actor(user_api_key_dict)
+    )
+    if updated is None:
+        raise _not_found(request.subscription_account_id)
+    return updated
+
+
+@router.post("/delete", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)])
+async def delete_subscription_account(
+    request: DeleteSubscriptionAccountRequest,
+    user_api_key_dict: Auth,
+    repository: Store,
+) -> LiteLLM_SubscriptionAccountTable:
+    """Stop tracking a subscription account's fee; its requests stay marked as subscription-covered."""
+    _require_proxy_admin(user_api_key_dict)
+    if await repository.find_by_id(request.subscription_account_id) is None:
+        raise _not_found(request.subscription_account_id)
+    deleted: Final = await repository.delete_account(request.subscription_account_id)
+    if deleted is None:
+        raise _not_found(request.subscription_account_id)
+    return deleted
+
+
+def _not_found(subscription_account_id: str) -> HTTPException:
+    return HTTPException(
+        status_code=status.HTTP_404_NOT_FOUND,
+        detail={"error": f"No subscription account with id {subscription_account_id}"},
+    )

litellm/proxy/management_endpoints/ui_sso.py

@@ -8,7 +8,6 @@ Has all /sso/* routes
 /sso/debug/callback - returns the OpenID object returned by the SSO provider
 """
 
-import asyncio
 import base64
 import hashlib
 import inspect
@@ -151,6 +150,7 @@ from litellm.proxy.utils import (
     get_custom_url,
     get_server_root_path,
 )
+from litellm.repositories.chunked_in import find_many_in
 from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import SSOConfigRepository
 from litellm.repositories.team_repository import TeamRepository
@@ -1855,7 +1855,7 @@ async def warn_if_id_jag_capture_gap(*, retention_enabled: RetentionCheck | None
     verbose_proxy_logger.warning("SSO debug callback ran with an oauth2_id_jag capture gap: %s", gap)
 
 
-async def create_team_member_add_task(team_id, user_info):
+async def create_team_member_add_task(team_id: str, user_info: NewUserResponse | LiteLLM_UserTable):
     """Create a task for adding a member to a team."""
     try:
         member: Final = Member(user_id=user_info.user_id, role="user")
@@ -1871,22 +1871,43 @@ async def create_team_member_add_task(team_id, user_info):
         verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e)
 
 
-async def add_missing_team_member(user_info: NewUserResponse | LiteLLM_UserTable, sso_teams: list[str]):
+async def find_existing_team_ids(team_ids: Sequence[str]) -> frozenset[str] | None:
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        return None
+    try:
+        rows: Final = await find_many_in(TeamRepository(prisma_client, use_writer=True).table, "team_id", team_ids)
+    except Exception as e:
+        verbose_proxy_logger.debug("[Non-Blocking] Error looking up SSO teams: %s", e)
+        return None
+    return frozenset(row.team_id for row in rows)
+
+
+async def add_missing_team_member(
+    user_info: NewUserResponse | LiteLLM_UserTable,
+    sso_teams: Sequence[str],
+    *,
+    add_team_member: Callable[
+        [str, NewUserResponse | LiteLLM_UserTable], Awaitable[object]
+    ] = create_team_member_add_task,
+    existing_team_ids: Callable[[Sequence[str]], Awaitable[frozenset[str] | None]] = find_existing_team_ids,
+) -> None:
     """
     - Get missing teams (diff b/w user_info.team_ids and sso_teams)
-    - Add missing user to missing teams
+    - Drop the IdP groups that are not teams with one lookup, since group claims often carry many of them
+    - Add missing user to missing teams, one at a time in sso_teams order, so user.teams keeps the claim's order
     """
-    # Handle None as empty list for new users
-    user_teams: Final = user_info.teams if user_info.teams is not None else []
-    missing_teams: Final = set(sso_teams) - set(user_teams)
-    missing_teams_list: Final = list(missing_teams)
-    tasks = []
-    tasks = [create_team_member_add_task(team_id, user_info) for team_id in missing_teams_list]
-
-    try:
-        await asyncio.gather(*tasks)
-    except Exception as e:
-        verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e)
+    held_teams: Final = frozenset(user_info.teams or ())
+    missing_teams: Final = tuple(team_id for team_id in dict.fromkeys(sso_teams) if team_id not in held_teams)
+    if not missing_teams:
+        return
+    existing: Final = await existing_team_ids(missing_teams)
+    teams_to_add: Final = (
+        missing_teams if existing is None else tuple(team_id for team_id in missing_teams if team_id in existing)
+    )
+    for team_id in teams_to_add:
+        await add_team_member(team_id, user_info)
 
 
 def get_disabled_non_admin_personal_key_creation():
@@ -2451,7 +2472,9 @@ async def fetch_cli_sso_team_details(
     except Exception as e:
         verbose_proxy_logger.error("Error fetching team details for CLI SSO session: %s", e)
         return None
-    return tuple(_cli_sso_team_detail(team_row.model_dump()) for team_row in prisma_teams)
+    fetched_details: Final = tuple(_cli_sso_team_detail(team_row.model_dump()) for team_row in prisma_teams)
+    details_by_team_id: Final = MappingProxyType({detail.team_id: detail for detail in fetched_details})
+    return tuple(details_by_team_id[team_id] for team_id in dict.fromkeys(teams) if team_id in details_by_team_id)
 
 
 def _cli_sso_session_teams(team_details: Sequence[CliSsoTeamDetail]) -> list[str]:

litellm/proxy/proxy_server.py

@@ -549,6 +549,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import (
     write_reload_interval,
 )
 from litellm.proxy.common_utils.proxy_state import ProxyState
+from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
 from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob
 from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState
 from litellm.proxy.common_utils.scheduled_job_stagger import (
@@ -717,6 +718,9 @@ from litellm.proxy.management_endpoints.customer_endpoints import (
     router as customer_router,
 )
 from litellm.proxy.management_endpoints.daily_activity_routes import router as daily_activity_router
+from litellm.proxy.management_endpoints.fallback_management_endpoints import (
+    apply_content_policy_fallbacks,
+)
 from litellm.proxy.management_endpoints.fallback_management_endpoints import (
     router as fallback_management_router,
 )
@@ -776,6 +780,9 @@ from litellm.proxy.management_endpoints.session_endpoints import (
     router as session_management_router,
 )
 from litellm.proxy.management_endpoints.sso.saml_sso import SAML_AUTHN_REQUEST_TTL_SECONDS
+from litellm.proxy.management_endpoints.subscription_accounts import (
+    router as subscription_accounts_router,
+)
 from litellm.proxy.management_endpoints.tag_management_endpoints import (
     router as tag_management_router,
 )
@@ -8159,14 +8166,38 @@ class ProxyConfig:
             if db_router_settings is not None and db_router_settings.param_value is not None
             else _EMPTY_SETTINGS_MAPPING
         )
+        previous_content_policy_rules: Final = self.router_settings.resolved().get("content_policy_fallbacks")
         self.router_settings.apply_db_row("router_settings", db_values)
         combined_router_settings: Final = self.router_settings.resolved()
         if combined_router_settings:
-            self._apply_router_settings(llm_router, combined_router_settings)
+            self._apply_router_settings(
+                llm_router,
+                combined_router_settings,
+                content_policy_owned_by_config=self.router_settings.owned_by_config("content_policy_fallbacks"),
+                content_policy_rules_changed=(
+                    combined_router_settings.get("content_policy_fallbacks") != previous_content_policy_rules
+                ),
+            )
 
     @staticmethod
-    def _apply_router_settings(llm_router: Router, router_settings: Mapping[str, object]) -> None:
-        llm_router.apply(ChangeSet(settings={k: v for k, v in router_settings.items() if k != "routing_groups"}))
+    def _apply_router_settings(
+        llm_router: Router,
+        router_settings: Mapping[str, object],
+        *,
+        content_policy_owned_by_config: bool,
+        content_policy_rules_changed: bool,
+    ) -> None:
+        llm_router.apply(
+            ChangeSet(
+                settings={
+                    k: v for k, v in router_settings.items() if k not in ("routing_groups", "content_policy_fallbacks")
+                }
+            )
+        )
+        if "content_policy_fallbacks" in router_settings and not content_policy_owned_by_config:
+            apply_content_policy_fallbacks(
+                llm_router, router_settings["content_policy_fallbacks"], log_skipped_rules=content_policy_rules_changed
+            )
         if "routing_groups" not in router_settings:
             return
         try:
@@ -8750,6 +8781,7 @@ class ProxyConfig:
         subscriber: Final = ConfigSyncSubscriber(
             redis_cache=redis_cache,
             resync_callbacks=(_resync_config_from_db,),
+            apply_model_ids=resync_model_deployments_by_ids,
         )
         self.config_sync_subscriber = subscriber
         subscriber.start()
@@ -21767,6 +21799,7 @@ app.include_router(memory_router)
 app.include_router(lens_router)
 app.include_router(plugin_router)
 app.include_router(cost_tracking_settings_router)
+app.include_router(subscription_accounts_router)
 app.include_router(prompt_caching_requests_router)
 app.include_router(router_settings_router)
 app.include_router(fallback_management_router)

litellm/proxy/schema.prisma

@@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
   @@id([event_id, target, version])
   @@index([target, state, next_attempt_at])
 }
+
+model LiteLLM_SubscriptionAccountTable {
+  subscription_account_id String @id @default(uuid())
+  custom_llm_provider String
+  account_id String
+  label String?
+  monthly_fee Float
+  currency String
+  billing_period_start String
+  created_at    DateTime               @default(now()) @map("created_at")
+  created_by String
+  updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")
+  updated_by String
+
+  @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
+}

litellm/proxy/spend_tracking/savings.py

@@ -689,9 +689,13 @@ def compute_savings_spend(
             cost_breakdown=cost_breakdown,
         )
     )
-    return SavingsSpend(
+    savings: Final = SavingsSpend(
         compression=compression,
         prompt_caching=prompt_caching,
         autorouter=0.0 if autorouter is None else autorouter,
         gateway_injected_caching=gateway_injected_caching,
     )
+    if all(isfinite(value) for value in savings):
+        return savings
+    verbose_proxy_logger.warning("Spend tracking - recording non-finite savings %s as 0 for model=%s", savings, model)
+    return SavingsSpend(*(value if isfinite(value) else 0.0 for value in savings))

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -912,7 +912,7 @@ def get_logging_payload(
             user=metadata.get("user_api_key_user_id", "") or "",
             team_id=metadata.get("user_api_key_team_id", "") or "",
             organization_id=metadata.get("user_api_key_org_id") or "",
-            metadata=safe_dumps(clean_metadata),
+            metadata=safe_dumps(clean_metadata, non_finite_as_null=True),
             cache_key=cache_key,
             spend=kwargs.get("response_cost", 0),
             total_tokens=usage.get("total_tokens", standard_logging_total_tokens),
@@ -1224,7 +1224,7 @@ def _get_messages_for_spend_logs_payload(
                 messages: Final = standard_logging_payload.get("messages")
                 if messages is not None:
                     try:
-                        return safe_dumps(messages)
+                        return safe_dumps(messages, non_finite_as_null=True)
                     except Exception:
                         return "{}"
     return "{}"
@@ -1919,7 +1919,8 @@ def _get_proxy_server_request_for_spend_logs_payload(
             _request_body_json_str: Final = safe_dumps(
                 _stored_request_without_requester_ip_address(stored_request_body)
                 if drop_requester_ip_address
-                else stored_request_body
+                else stored_request_body,
+                non_finite_as_null=True,
             )
             if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str:
                 verbose_proxy_logger.info(
@@ -2000,7 +2001,7 @@ def _get_response_for_spend_logs_payload(
         if isinstance(sanitized_response, str):
             result_str = strip_null_bytes(sanitized_response)
         else:
-            result_str = safe_dumps(sanitized_response)
+            result_str = safe_dumps(sanitized_response, non_finite_as_null=True)
         if LITELLM_TRUNCATED_PAYLOAD_FIELD in result_str:
             verbose_proxy_logger.info(
                 "Spend Log: response was truncated before storing in DB. %s",
@@ -2071,7 +2072,7 @@ def spend_log_row_with_retained_metadata(
         return row
     metadata: Final = _SPEND_LOGS_METADATA_ADAPTER.validate_json(metadata_json)
     retained: Final = {name: value for name, value in metadata.items() if fields.keeps(name)}
-    return {**row, "metadata": safe_dumps(retained)}
+    return {**row, "metadata": safe_dumps(retained, non_finite_as_null=True)}
 
 
 def _get_status_for_spend_log(

litellm/proxy/spend_tracking/subscription_billing_periods.py

@@ -0,0 +1,54 @@
+from calendar import monthrange
+from dataclasses import dataclass
+from datetime import date, timedelta
+from typing import Final
+
+
+@dataclass(frozen=True, slots=True)
+class BillingPeriodRange:
+    start: date
+    end: date
+
+
+def _month_index(anchor: date, months_after_anchor: int) -> int:
+    return anchor.year * 12 + (anchor.month - 1) + months_after_anchor
+
+
+_LAST_MONTH_INDEX: Final = _month_index(date.max, 0)
+
+
+def _month_start(anchor: date, months_after_anchor: int) -> date:
+    month_index: Final = _month_index(anchor, months_after_anchor)
+    year: Final = month_index // 12
+    month: Final = month_index % 12 + 1
+    return date(year, month, min(anchor.day, monthrange(year, month)[1]))
+
+
+def _period_end(anchor: date, months_after_anchor: int) -> date:
+    if _month_index(anchor, months_after_anchor + 1) > _LAST_MONTH_INDEX:
+        return date.max
+    return _month_start(anchor, months_after_anchor + 1) - timedelta(days=1)
+
+
+def _first_candidate_index(anchor: date, start_date: date) -> int:
+    months_apart: Final = (start_date.year - anchor.year) * 12 + (start_date.month - anchor.month)
+    return max(0, months_apart - 1)
+
+
+def billing_periods_starting_within(anchor: date, start_date: date, end_date: date) -> tuple[BillingPeriodRange, ...]:
+    """Monthly periods anchored on `anchor`'s day of month whose start lies inside [start_date, end_date]."""
+    if end_date < start_date or end_date < anchor:
+        return ()
+    first_index: Final = _first_candidate_index(anchor, start_date)
+    last_index: Final = min(
+        first_index + _month_span(start_date, end_date) + 1, _LAST_MONTH_INDEX - _month_index(anchor, 0)
+    )
+    candidates: Final = (
+        BillingPeriodRange(start=_month_start(anchor, index), end=_period_end(anchor, index))
+        for index in range(first_index, last_index + 1)
+    )
+    return tuple(period for period in candidates if start_date <= period.start <= end_date)
+
+
+def _month_span(start_date: date, end_date: date) -> int:
+    return (end_date.year - start_date.year) * 12 + (end_date.month - start_date.month)