Merge remote-tracking branch 'origin/main' into litellm_ollama_model_info_api_base

# Conflicts:
#	litellm/constants.py

.github/workflows/test-unit.yml

@@ -227,6 +227,7 @@ jobs:
               tests/unit/llms/test_oss_decision.py
               tests/unit/llms/test_polling_url_origin_match.py
               tests/unit/llms/test_predibase_transformation.py
+              tests/unit/llms/test_subscription_accounts.py
               tests/unit/llms/a2a
               tests/unit/llms/aiml
               tests/unit/llms/aiohttp_openai

backend/routes/allowlist.py

@@ -55,6 +55,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/cache_settings",
     "/coordination_redis/",
     "/cost_tracking",
+    "/subscription_accounts/",
     "/cost_optimization/",
     "/cost/",
     "/credentials",

litellm-proxy-extras/litellm_proxy_extras/migrations/20261028000000_subscription_accounts/migration.sql

@@ -0,0 +1,21 @@
+SET LOCAL lock_timeout = '10s';
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_SubscriptionAccountTable" (
+    "subscription_account_id" TEXT NOT NULL,
+    "custom_llm_provider" TEXT NOT NULL,
+    "account_id" TEXT NOT NULL,
+    "label" TEXT,
+    "monthly_fee" DOUBLE PRECISION NOT NULL,
+    "currency" TEXT NOT NULL,
+    "billing_period_start" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT NOT NULL,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_by" TEXT NOT NULL,
+
+    CONSTRAINT "LiteLLM_SubscriptionAccountTable_pkey" PRIMARY KEY ("subscription_account_id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SubscriptionAccountTable_provider_account_key" ON "LiteLLM_SubscriptionAccountTable"("custom_llm_provider", "account_id");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
   @@id([event_id, target, version])
   @@index([target, state, next_attempt_at])
 }
+
+model LiteLLM_SubscriptionAccountTable {
+  subscription_account_id String @id @default(uuid())
+  custom_llm_provider String
+  account_id String
+  label String?
+  monthly_fee Float
+  currency String
+  billing_period_start String
+  created_at    DateTime               @default(now()) @map("created_at")
+  created_by String
+  updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")
+  updated_by String
+
+  @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
+}

litellm/constants.py

@@ -68,6 +68,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
         "retry_after",
         "fallbacks",
         "context_window_fallbacks",
+        "content_policy_fallbacks",
         "retry_policy",
         "model_group_retry_policy",
         "model_group_alias",
@@ -420,6 +421,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
     "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
 )
 BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
+REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
+REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0
 
 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
 WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
@@ -899,6 +902,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
 
 PROVIDERS_THAT_SEND_KEYS_ON_MODEL_INFO_LOOKUPS: Final = frozenset({"ollama", "ollama_chat", "lemonade"})
 
+SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
 A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
 
 LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [

litellm/integrations/anthropic_cache_control_hook.py

@@ -116,12 +116,12 @@ def configured_injection_points(value: object) -> Sequence[CacheControlInjection
     return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict))
 
 
-def _served_provider_for_model(model: str) -> str | None:
+def _served_provider_for_model(model: str, api_base: object = None) -> str | None:
     from litellm.exceptions import BadRequestError
     from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 
     try:
-        _, provider, _, _ = get_llm_provider(model=model)
+        _, provider, _, _ = get_llm_provider(model=model, api_base=api_base if isinstance(api_base, str) else None)
     except BadRequestError:
         return None
     return provider
@@ -536,7 +536,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if model is None:
             return False
         hosted_flag: Final = _hosted_openai_dialect_flag(
-            model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider
+            model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
         )
         if hosted_flag is not None:
             return hosted_flag
@@ -559,8 +559,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
 
     @staticmethod
-    def _resolve_provider(model: str) -> str | None:
-        return _served_provider_for_model(model)
+    def _resolve_provider(model: str, api_base: object = None) -> str | None:
+        return _served_provider_for_model(model, api_base)
 
     @staticmethod
     def count_request_cache_breakpoints(

litellm/integrations/otel/model/spans.py

@@ -361,6 +361,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_LensReview",
         "LiteLLM_LensIngestionKey",
         "LiteLLM_LensWorker",
+        "LiteLLM_SubscriptionAccountTable",
         "LiteLLM_LensSignalConfig",
         "LiteLLM_LensTraceSignal",
     )

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
     """
     from litellm.litellm_core_utils.token_counter import get_image_type
 
+    media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
+
     # If we have a valid content type that's not generic, use it
-    if current_content_type and current_content_type not in [
+    if media_type and media_type not in [
         "binary/octet-stream",
         "application/octet-stream",
     ]:
-        return current_content_type
+        return media_type
 
     # Extension to MIME type mapping
     # Supports images, documents, and other common file types

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -912,9 +912,9 @@ def create_anthropic_image_param(
     image_url_input: str | dict,
     format: str | None = None,
     is_bedrock_invoke: bool = False,
-) -> AnthropicMessagesImageParam:
+) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
     """
-    Create an AnthropicMessagesImageParam from an image URL input.
+    Create an Anthropic content block from an image URL input.
 
     Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
     """
@@ -933,14 +933,7 @@ def create_anthropic_image_param(
         if is_bedrock_invoke or image_url.startswith("http://"):
             base64_url: Final = convert_url_to_base64(url=image_url)
             image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
-            return AnthropicMessagesImageParam(
-                type="image",
-                source=AnthropicContentParamSource(
-                    type="base64",
-                    media_type=image_chunk["media_type"],
-                    data=image_chunk["data"],
-                ),
-            )
+            return _anthropic_content_element_factory(image_chunk)
         else:
             # HTTPS URL - pass directly for regular Anthropic
             return AnthropicMessagesImageParam(
@@ -953,14 +946,7 @@ def create_anthropic_image_param(
     else:
         # Convert to base64 for data URIs or other formats
         image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
-        return AnthropicMessagesImageParam(
-            type="image",
-            source=AnthropicContentParamSource(
-                type="base64",
-                media_type=image_chunk["media_type"],
-                data=image_chunk["data"],
-            ),
-        )
+        return _anthropic_content_element_factory(image_chunk)
 
 
 # The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
                         anthropic_content_element=_anthropic_image_param,
                         original_content_element=content,
                     )
-                    anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
+                    anthropic_content_list.append(
+                        cast(
+                            AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
+                            _anthropic_image_param,
+                        )
+                    )
             elif content["type"] == "tool_reference":
                 anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
             elif content["type"] == "file":

litellm/litellm_core_utils/realtime_streaming.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import time
 import traceback
 from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
 from dataclasses import dataclass
@@ -15,13 +16,19 @@ from litellm._logging import redact_internal_details_from_client_message, verbos
 from litellm._uuid import uuid
 from litellm.constants import (
     BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
+    REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
+    REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
     REALTIME_METERED_USAGE_KEY,
     REALTIME_SESSION_FAILURE_LOGGED_KEY,
     REALTIME_SESSION_SUCCESS_LOGGED_KEY,
 )
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
-from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
+from litellm.llms.base_llm.realtime.transformation import (
+    BaseRealtimeConfig,
+    RealtimeBackend,
+    RealtimeSessionUpdateRejection,
+)
 from litellm.types.llms.openai import (
     OpenAIRealtimeEvents,
     OpenAIRealtimeOutputItemDone,
@@ -125,6 +132,65 @@ class _LoggingWorker(Protocol):
 _DECODED_JSON: Final = TypeAdapter(object)
 
 
+_SESSION_FIELDS: Final = TypeAdapter(dict[str, object])
+_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+_SESSION_MERGE_MAX_DEPTH: Final = 8
+
+
+class _SessionUpdateFrame(TypedDict):
+    type: ReadOnly[str]
+    session: ReadOnly[Mapping[str, object]]
+
+
+def _merge_session_fields(
+    base: Mapping[str, object], update: Mapping[str, object], depth: int = 0
+) -> Mapping[str, object]:
+    def merged_value(key: str) -> object:
+        if key not in update:
+            return base[key]
+        value: Final = update[key]
+        base_value: Final = base.get(key)
+        if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping):
+            return _merge_session_fields(
+                _SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1
+            )
+        return update[key]
+
+    return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))})
+
+
+def _session_update_frame(session: Mapping[str, object]) -> str:
+    frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session}
+    return json.dumps(frame, default=_SESSION_FIELDS.validate_python)
+
+
+_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection(
+    code="session_update_too_large",
+    message=(
+        "The session.update frames sent before the first conversation content add up to more than 10 MB, "
+        "so this session.update was not applied"
+    ),
+    param="session",
+)
+
+
+def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, object]:
+    return {
+        "type": "error",
+        "event_id": f"event_{uuid.uuid4()}",
+        "error": {
+            "type": "invalid_request_error",
+            "code": "audio_input_not_screenable",
+            "message": (
+                "A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it "
+                "starts answering it, so audio input was not sent. Send text with conversation.item.create instead"
+            ),
+            "param": None,
+            "event_id": client_event_id if isinstance(client_event_id, str) else None,
+        },
+    }
+
+
 def _decode_json_object(payload: str) -> Mapping[str, object]:
     """Decode a realtime frame into its top-level field mapping."""
     return json.loads(payload)
@@ -243,8 +309,17 @@ class RealTimeStreaming:
         event_normalizer: RealtimeEventNormalizer | None = None,
         usage_meter: RealtimeUsageMeter | None = None,
         logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
+        lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
+        lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
+        clock: Callable[[], float] = time.monotonic,
     ):
         self.websocket: _ClientWebSocket = websocket
+        self._clock: Final = clock
+        self._lazy_setup_deadline: Final = clock() + lazy_setup_deadline_seconds
+        self._lazy_setup_content_grace_seconds: Final = lazy_setup_content_grace_seconds
+        self._held_client_content: tuple[str, ...] = ()
+        self._held_client_content_bytes: int = 0
+        self._held_client_content_deadline: float | None = None
         self._replay_scope: Final = client_websocket_scope(websocket)
         self.backend_ws = backend_ws
         self.logging_obj = logging_obj
@@ -300,6 +375,10 @@ class RealTimeStreaming:
         # Gemini Live rejects a follow-up BidiGenerateContentSetup once any
         # content (realtimeInput / clientContent / toolResponse) has been sent.
         self._content_sent_after_setup: bool = False
+        self._held_session_update: Mapping[str, object] | None = None
+        self._applied_session_fields: Mapping[str, object] | None = None
+        self._held_session_update_bytes: int = 0
+        self._unscreened_audio_refused: bool = False
         # Whether this is a transcription-only session (session.type == "transcription",
         # e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and
         # their input_audio_transcription.completed usage drives duration-based cost.
@@ -337,6 +416,17 @@ class RealTimeStreaming:
         ]
     )
     _CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"])
+    _CLIENT_CONTENT_TYPES = frozenset(
+        [
+            "conversation.item.create",
+            "response.create",
+            "input_audio_buffer.append",
+            "input_audio_buffer.commit",
+            "input_audio_buffer.end",
+        ]
+    )
+    _HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create", "input_audio_buffer.append"])
+    _DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"])
     _AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
         "pcm16": {"type": "audio/pcm", "rate": 24000},
         "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
@@ -377,7 +467,7 @@ class RealTimeStreaming:
             message_obj: dict[str, Any] = cast(dict[str, Any], message)
         else:
             message_obj = cast(dict[str, Any], json.loads(message))
-        self._collect_tool_calls_from_response_done(cast(dict, message_obj))
+        self._collect_tool_calls_from_response_done(message_obj)
         if not self._should_store_message(message_obj):
             return
         try:
@@ -502,7 +592,7 @@ class RealTimeStreaming:
         self.store_message(flush_event)
         self._capture_transcription_usage(flush_event)
 
-    def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
+    def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None:
         """Extract function_call items from response.done events for spend logging."""
         try:
             if event_obj.get("type") != "response.done":
@@ -607,6 +697,167 @@ class RealTimeStreaming:
                 self._usage_meter.observe_client_event(sent_event)
         return True
 
+    def _builds_setup_lazily(self) -> bool:
+        if self.provider_config is None:
+            return False
+        return self.provider_config.builds_setup_from_session_update()
+
+    def _holds_session_updates_until_first_content(self) -> bool:
+        return self.session_configuration_request is None and self._builds_setup_lazily()
+
+    async def _rejected_setup_session_update(self, message: str) -> bool:
+        provider_config: Final = self.provider_config
+        if provider_config is None or not provider_config.builds_setup_from_session_update():
+            return False
+        client_event: Final = _decode_json_object(message)
+        session: Final = client_event.get("session")
+        update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
+        if self._leaves_applied_setup_unchanged(update):
+            await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS)
+            return True
+        rejection: Final = provider_config.setup_session_update_rejection(
+            self.model, update, setup_sent=self.session_configuration_request is not None
+        )
+        if rejection is None:
+            return False
+        await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id"))))
+        return True
+
+    def _held_session_update_message(self) -> str:
+        return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS)
+
+    async def _hold_session_update(self, message: str) -> None:
+        client_event: Final = _decode_json_object(message)
+        if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES:
+            await self.websocket.send_text(
+                json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id")))
+            )
+            return
+        session: Final = client_event.get("session")
+        update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
+        held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
+        self._held_session_update = held
+        self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8"))
+        await self._acknowledge_session_setup(held)
+
+    def _setup_preview(self, session: Mapping[str, object]) -> str | None:
+        if self.provider_config is None:
+            return None
+        preview: Final = self.provider_config.transform_realtime_request(
+            _session_update_frame(session), self.model, None
+        )
+        return next(
+            (frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
+        )
+
+    def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
+        applied: Final = self._applied_session_fields
+        if self.session_configuration_request is None or applied is None:
+            return False
+        applied_setup: Final = self._setup_preview(applied)
+        merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update))
+        return (
+            applied_setup is not None
+            and merged_setup is not None
+            and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
+        )
+
+    async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
+        if self.provider_config is None:
+            return
+        setup: Final = self._setup_preview(session)
+        if setup is None:
+            return
+        acknowledged: Final = self.provider_config.transform_session_created_event(
+            self.model, str(self.logging_obj.litellm_trace_id), setup
+        )
+        if acknowledged is None:
+            return
+        event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"}
+        event_str: Final = json.dumps(event)
+        self.store_message(event_str)
+        await self._send_event_to_client(event, event_str)
+
+    async def _send_held_session_update_as_setup(self) -> None:
+        held: Final = self._held_session_update
+        if held is None:
+            return
+        message: Final = self._held_session_update_message()
+        self._held_session_update = None
+        self._held_session_update_bytes = 0
+        if await self._send_to_backend(message):
+            self._applied_session_fields = held
+            await self._send_held_client_content()
+
+    def _holds_client_content(self, msg_type: str | None, message: str) -> bool:
+        held_types: Final = (
+            RealTimeStreaming._DEFERRED_SETUP_HELD_CONTENT_TYPES
+            if self._uses_deferred_backend_setup()
+            else RealTimeStreaming._HELD_CONTENT_TYPES
+        )
+        return (
+            msg_type in held_types
+            and self._held_session_update is not None
+            and self._holds_session_updates_until_first_content()
+            and not self._pending_messages_until_setup
+            and len(self._held_client_content) < RealTimeStreaming._MAX_BUFFERED_MESSAGES
+            and self._held_client_content_bytes + len(message.encode("utf-8")) <= RealTimeStreaming._MAX_BUFFERED_BYTES
+        )
+
+    def _hold_client_content(self, message: str) -> None:
+        if self._held_client_content_deadline is None:
+            self._held_client_content_deadline = self._clock() + self._lazy_setup_content_grace_seconds
+        self._held_client_content = (*self._held_client_content, message)
+        self._held_client_content_bytes += len(message.encode("utf-8"))
+
+    async def _send_held_client_content(self) -> None:
+        held: Final = self._held_client_content
+        self._held_client_content = ()
+        self._held_client_content_bytes = 0
+        self._held_client_content_deadline = None
+        for message in held:
+            await self._send_to_backend(message)
+
+    def _setup_deadline(self) -> float:
+        content_deadline: Final = self._held_client_content_deadline
+        return (
+            self._lazy_setup_deadline if content_deadline is None else min(self._lazy_setup_deadline, content_deadline)
+        )
+
+    async def _send_setup_before_content(self, msg_type: str | None) -> None:
+        if self._holds_session_updates_until_first_content():
+            await self._send_held_session_update_as_setup()
+        await self._send_default_setup_before_first_client_message(msg_type)
+
+    async def _receive_client_message_before_setup_deadline(self) -> str | None:
+        if not self._holds_session_updates_until_first_content():
+            return await self._receive_client_message()
+        receive: Final = asyncio.ensure_future(self._receive_client_message())
+        try:
+            done, _ = await asyncio.wait((receive,), timeout=self._setup_deadline() - self._clock())
+            if not done:
+                await self._send_setup_before_content(None)
+            return await receive
+        finally:
+            receive.cancel()
+
+    async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None:
+        if self.session_configuration_request is not None or self.provider_config is None:
+            return
+        if msg_type == "session.update" or (
+            self._uses_deferred_backend_setup()
+            and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup)
+        ):
+            return
+        default_setup: Final = self.provider_config.session_configuration_request(self.model)
+        if not default_setup:
+            return
+        setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup)
+        await self.backend_ws.send(setup)
+        self._cache_session_configuration_request(setup)
+        self._applied_session_fields = _NO_SESSION_FIELDS
+        await self._send_held_client_content()
+
     def _enforce_transcription_session_model(self, message: str) -> str:
         """Force client transcription session updates to the authorized model.
 
@@ -851,6 +1102,20 @@ class RealTimeStreaming:
         self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
         await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
 
+    async def _send_blocked_turn_to_backend(self, pre_block_backend_message: str | None, item_id: str | None) -> None:
+        if pre_block_backend_message is None and self._is_transcription_session:
+            return
+        await self._send_setup_before_content("conversation.item.create")
+        # Deliver any caller-supplied backend message FIRST so that
+        # protocol contracts requiring a specific ordering (e.g.
+        # Gemini Live's mandatory ``toolResponse`` after a
+        # ``toolCall``) are honored before the guardrail's own
+        # clientContent / cancel messages are sent.
+        if pre_block_backend_message is not None:
+            await self._send_to_backend(pre_block_backend_message)
+        if not self._is_transcription_session:
+            await self._withdraw_blocked_turn(item_id)
+
     async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
         await self._send_guardrail_cancel()
         if item_id is not None and self._deletes_conversation_items:
@@ -958,6 +1223,21 @@ class RealTimeStreaming:
         """Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
         return not self._is_transcription_session and self._has_audio_transcription_guardrails()
 
+    def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
+        return (
+            msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES
+            and self._builds_setup_lazily()
+            and self._should_disable_vad_auto_response()
+        )
+
+    async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None:
+        if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES:
+            return
+        self._unscreened_audio_refused = True
+        await self.websocket.send_text(
+            json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id")))
+        )
+
     async def run_realtime_guardrails(
         self,
         transcript: str,
@@ -1030,15 +1310,7 @@ class RealTimeStreaming:
                 # Use realtime_violation_message if configured; fall back to guardrail error text.
                 error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
 
-                # Deliver any caller-supplied backend message FIRST so that
-                # protocol contracts requiring a specific ordering (e.g.
-                # Gemini Live's mandatory ``toolResponse`` after a
-                # ``toolCall``) are honored before the guardrail's own
-                # clientContent / cancel messages are sent.
-                if pre_block_backend_message is not None:
-                    await self._send_to_backend(pre_block_backend_message)
-                if not self._is_transcription_session:
-                    await self._withdraw_blocked_turn(item_id)
+                await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
                 # Send the policy violation hint (shows as small gray status text in UI).
                 await self.websocket.send_text(
                     json.dumps(
@@ -1136,6 +1408,9 @@ class RealTimeStreaming:
                                 break
                     finally:
                         self._flushing_pending_messages_until_setup = False
+                if self.session_created_sent_to_client and self._builds_setup_lazily():
+                    await self._maybe_send_guardrail_turn_detection_update()
+                    continue
                 if self.session_created_sent_to_client:
                     # A synthetic session.created (with placeholder defaults) was
                     # already forwarded to the client when we connected.  The
@@ -1295,7 +1570,11 @@ class RealTimeStreaming:
             return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
 
     def _backend_refused_session(self, close: BackendClose) -> bool:
-        return close.code != 1000 and not self._backend_sent_frames
+        return (
+            close.code != 1000
+            and not self._backend_sent_frames
+            and not self._holds_session_updates_until_first_content()
+        )
 
     async def _log_session_end(self, transport_refusal: Exception | None) -> None:
         provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
@@ -1488,7 +1767,7 @@ class RealTimeStreaming:
         client_event: _ClientEventFrame
         try:
             while True:
-                message = await self._receive_client_message()
+                message = await self._receive_client_message_before_setup_deadline()
                 if message is None:
                     return ClientLoopExit.CLIENT_DISCONNECTED
 
@@ -1600,6 +1879,7 @@ class RealTimeStreaming:
                     if (
                         msg_type == "session.update"
                         and self.session_configuration_request is None
+                        and self._held_session_update is None
                         and not self._guardrail_turn_detection_update_sent
                         and self._should_disable_vad_auto_response()
                         and (
@@ -1663,12 +1943,25 @@ class RealTimeStreaming:
                 except (json.JSONDecodeError, AttributeError):
                     pass
 
+                if self._refuses_unscreened_audio(msg_type):
+                    await self._refuse_unscreened_audio(msg_type, message)
+                    continue
+
                 ## LOGGING
                 # Log after any in-place modifications (GA remap, guardrail
                 # turn_detection injection) so audit logs reflect what we
                 # actually forward to the backend.
                 self.store_input(message=message)
 
+                if self._holds_client_content(msg_type, message):
+                    self._hold_client_content(message)
+                    continue
+                if self._held_client_content and msg_type != "session.update":
+                    await self._send_setup_before_content(msg_type)
+
+                if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup():
+                    await self._send_setup_before_content(msg_type)
+
                 if self._should_buffer_client_message_until_setup(message):
                     self._buffer_pending_message_until_setup(message)
                     continue
@@ -1697,6 +1990,15 @@ class RealTimeStreaming:
                 # ``_maybe_send_guardrail_turn_detection_update`` would retry.
                 if msg_type == "session.update" and self.provider_config is None:
                     await self._send_session_gate_once()
+                if msg_type == "session.update" and await self._rejected_setup_session_update(message):
+                    continue
+                if msg_type == "session.update" and self._holds_session_updates_until_first_content():
+                    await self._hold_session_update(message)
+                    if self._pending_messages_until_setup:
+                        await self._send_held_session_update_as_setup()
+                    continue
+                if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES:
+                    await self._send_setup_before_content(msg_type)
                 try:
                     sent = await self._send_to_backend(message)
                 except RealtimeTranscriptionProtocolError as e:

litellm/litellm_core_utils/streaming_handler.py

@@ -40,6 +40,7 @@ from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import (
+    CHOICE_LOGPROBS_OR_JSON,
     CacheCreationTokenDetails,
     CompletionTokensDetailsWrapper,
     Delta,
@@ -1201,7 +1202,10 @@ class CustomStreamWrapper:
                                         choice_json.pop(
                                             "finish_reason", None
                                         )  # for mistral etc. which return a value in their last chunk (not-openai compatible).
-                                        choices.append(StreamingChoices(**choice_json))
+                                        choice_json.pop("logprobs", None)
+                                        choices.append(
+                                            StreamingChoices(**choice_json, logprobs=getattr(choice, "logprobs", None))
+                                        )
                                 except Exception:
                                     choices.append(StreamingChoices())
                             setattr(model_response, "choices", choices)
@@ -1605,7 +1609,7 @@ class CustomStreamWrapper:
                     model_response.system_fingerprint = response_obj["original_chunk"].system_fingerprint
                     self.system_fingerprint = response_obj["original_chunk"].system_fingerprint
             if response_obj["logprobs"] is not None:
-                model_response.choices[0].logprobs = response_obj["logprobs"]
+                model_response.choices[0].logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(response_obj["logprobs"])
 
             if response_obj["usage"] is not None:
                 if isinstance(response_obj["usage"], dict):

litellm/llms/azure/responses/transformation.py

@@ -14,6 +14,7 @@ from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
 from litellm.llms.azure.common_utils import BaseAzureLLM
+from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.types.llms.openai import *
 from litellm.types.responses.main import *
@@ -92,6 +93,12 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
         return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)
 
+    def deployment_names_wrap_openai_models(self) -> bool:
+        return True
+
+    def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
+        return SUMMARY_ONLY_REASONING_FIELD_VALUES if self._is_gpt_5_model(model) else None
+
     def get_supported_openai_params(self, model: str) -> list:
         """
         Azure Responses API does not support context_management (compaction).

litellm/llms/base_llm/realtime/transformation.py

@@ -1,10 +1,12 @@
+import dataclasses
 from abc import ABC, abstractmethod
 from collections.abc import Mapping, Sequence
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, Protocol
+from typing import TYPE_CHECKING, Any, Literal, Protocol
 
 import httpx
 
+from litellm._uuid import uuid
 from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
 from litellm.types.realtime import (
     RealtimeInputAudioTranscriptionUsage,
@@ -22,6 +24,26 @@ else:
     LiteLLMLoggingObj = Any
 
 
+@dataclasses.dataclass(frozen=True, slots=True)
+class RealtimeSessionUpdateRejection:
+    code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"]
+    message: str
+    param: str
+
+    def error_event(self, client_event_id: object) -> dict[str, object]:
+        return {
+            "type": "error",
+            "event_id": f"event_{uuid.uuid4()}",
+            "error": {
+                "type": "invalid_request_error",
+                "code": self.code,
+                "message": self.message,
+                "param": self.param,
+                "event_id": client_event_id if isinstance(client_event_id, str) else None,
+            },
+        }
+
+
 class RealtimeBackend(Protocol):
     async def __aenter__(self) -> object: ...
 
@@ -93,6 +115,14 @@ class BaseRealtimeConfig(ABC):
     def session_configuration_request(self, model: str) -> str | None:  # message sent to setup the realtime session
         return None
 
+    def builds_setup_from_session_update(self) -> bool:
+        return False
+
+    def setup_session_update_rejection(
+        self, model: str, session: Mapping[str, object], *, setup_sent: bool
+    ) -> RealtimeSessionUpdateRejection | None:
+        return None
+
     def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
         return None
 

litellm/llms/base_llm/responses/transformation.py

@@ -1,7 +1,7 @@
 import types
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast, get_args
 
 import httpx
 
@@ -27,6 +27,20 @@ else:
     BaseLLMException = Any
 
 ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"]
+ReasoningFieldValues: TypeAlias = Mapping[str, frozenset[str | None] | None]
+REASONING_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
+SUMMARY_ONLY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
+    {"summary": REASONING_SUMMARY_VALUES}
+)
+STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
+    {**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": frozenset({"standard"})}
+)
+STANDARD_MODE_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
+    {**STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, "context": None}
+)
+ANY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = types.MappingProxyType(
+    {**SUMMARY_ONLY_REASONING_FIELD_VALUES, "mode": None, "context": None}
+)
 
 
 class BaseResponsesAPIConfig(ABC):
@@ -100,6 +114,18 @@ class BaseResponsesAPIConfig(ABC):
         """The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for."""
         return "detailed"
 
+    def reasoning_field_values(self) -> ReasoningFieldValues | None:
+        return None
+
+    def deployment_names_wrap_openai_models(self) -> bool:
+        return False
+
+    def unresolved_model_reasoning_field_values(self, model: str) -> ReasoningFieldValues | None:
+        return None
+
+    def ignores_unsupported_reasoning_fields(self) -> bool:
+        return False
+
     @abstractmethod
     def map_openai_params(
         self,

litellm/llms/bedrock/realtime/handler.py

@@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm._logging import redact_string, verbose_proxy_logger
-from litellm._uuid import uuid
 from litellm.constants import (
     BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
     BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
@@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
+from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection
 from litellm.types.llms.bedrock import AwsAuthParams
 from litellm.types.llms.openai import OpenAIRealtimeEvents
 from litellm.types.realtime import RealtimeResponseTransformInput
 
 from ..base_aws_llm import BaseAWSLLM, run_aws_signing
 from ..common_utils import BedrockError
-from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
+from .transformation import BedrockRealtimeConfig, requested_modalities
 
 _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
 
@@ -256,23 +256,11 @@ async def _ack_session_update(
 async def _reject_session_update(
     client_ws: RealtimeClientWebSocket,
     parsed_client_message: Mapping[str, JsonValue],
-    rejection: BedrockSessionUpdateRejection,
+    rejection: RealtimeSessionUpdateRejection,
 ) -> bool:
     """Send an error event for a session.update Bedrock cannot apply; False means the client is gone."""
-    client_event_id: Final = parsed_client_message.get("event_id")
-    error_event: Final = {
-        "type": "error",
-        "event_id": f"event_{uuid.uuid4()}",
-        "error": {
-            "type": "invalid_request_error",
-            "code": rejection.code,
-            "message": rejection.message,
-            "param": rejection.param,
-            "event_id": client_event_id if isinstance(client_event_id, str) else None,
-        },
-    }
     try:
-        await client_ws.send_text(json.dumps(error_event))
+        await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id"))))
     except Exception as e:  # noqa: BLE001  # any send failure means the client is gone
         verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
         return False

litellm/llms/bedrock/realtime/transformation.py

@@ -11,7 +11,7 @@ import math
 import uuid as uuid_lib
 from collections.abc import Mapping, Sequence
 from types import MappingProxyType
-from typing import Final, Literal
+from typing import Final
 
 import httpx
 from pydantic import ConfigDict, JsonValue, TypeAdapter
@@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
+from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
 from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
 from litellm.types.llms.base import LiteLLMBaseModel
@@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[
     return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
 
 
-@dataclasses.dataclass(frozen=True, slots=True)
-class BedrockSessionUpdateRejection:
-    code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
-    message: str
-    param: str
-
-
 @dataclasses.dataclass(frozen=True, slots=True)
 class BedrockLockedSessionSettings:
     """The session settings Bedrock Nova Sonic locks in once the prompt starts."""
@@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             max_tokens=max_tokens,
         )
 
-    def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None:
+    def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None:
         """The error for a session.update this session cannot apply, or None when it applies."""
         session_config: Final = json_message.get("session")
         if not isinstance(session_config, Mapping):
@@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         malformed: Final = () if self.drop_params else _malformed_settings(session_config)
         if malformed:
             name, expected = malformed[0]
-            return BedrockSessionUpdateRejection(
+            return RealtimeSessionUpdateRejection(
                 code="invalid_type",
                 message=(
                     f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
@@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             )
         unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
         if unplayable_voice is not None:
-            return BedrockSessionUpdateRejection(
+            return RealtimeSessionUpdateRejection(
                 code="invalid_value",
                 message=(
                     f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
@@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         if not self.prompt_started:
             return None
         if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings:
-            return BedrockSessionUpdateRejection(
+            return RealtimeSessionUpdateRejection(
                 code="session_update_not_applied",
                 message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied",
                 param="session",

litellm/llms/chatgpt/authenticator.py

@@ -122,6 +122,12 @@ class Authenticator:
         return tokens["access_token"]
 
     def get_account_id(self, access_token: str | None = None) -> str | None:
+        return self._account_id(access_token, persist_derived=True)
+
+    def read_account_id(self, access_token: str | None = None) -> str | None:
+        return self._account_id(access_token, persist_derived=False)
+
+    def _account_id(self, access_token: str | None, persist_derived: bool) -> str | None:
         auth_data: Final = self._read_auth_file()
         if access_token and (not auth_data or auth_data.get("access_token") != access_token):
             return extract_chatgpt_account_id(access_token)
@@ -133,7 +139,7 @@ class Authenticator:
         id_token: Final = auth_data.get("id_token")
         saved_access_token: Final = auth_data.get("access_token")
         derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token))
-        if derived:
+        if derived and persist_derived:
             self._write_auth_file({**auth_data, "account_id": derived})
         return derived
 

litellm/llms/chatgpt/responses/transformation.py

@@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
     safe_convert_created_field,
 )
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.custom_httpx.sse_lines import split_sse_lines
 from litellm.llms.openai.common_utils import OpenAIError
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
@@ -295,3 +296,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def supports_native_websocket(self) -> bool:
         """ChatGPT does not support native WebSocket for Responses API"""
         return False
+
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return STANDARD_MODE_REASONING_FIELD_VALUES

litellm/llms/custom_httpx/llm_http_handler.py

@@ -6973,14 +6973,19 @@ class BaseLLMHTTPHandler:
                     ),
                 )
 
-                # Auto-send session setup if the provider requires it (e.g.
-                # Gemini/Vertex AI Live needs a `setup` before any realtime_input).
+                # Auto-send session setup at connect unless the provider builds it from the
+                # client's first session.update (Gemini/Vertex AI Live accept exactly one
+                # setup, so RealTimeStreaming sends theirs on the first client message).
                 # Build the streaming handler first so a transcription guardrail's
                 # auto-response disable can be folded into this one setup: Gemini
                 # rejects a second setup, so a follow-up disable would be dropped
                 # and the guardrail bypassed.
                 _session_config: str | None = None
-                if provider_config.requires_session_configuration():
+                sends_setup_at_connect: Final = (
+                    provider_config.requires_session_configuration()
+                    and not provider_config.builds_setup_from_session_update()
+                )
+                if sends_setup_at_connect:
                     _session_config = provider_config.session_configuration_request(model)
                     if _session_config:
                         _session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable(

litellm/llms/fireworks_ai/responses/transformation.py

@@ -6,6 +6,10 @@ from urllib.parse import unquote
 import httpx
 
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.base_llm.responses.transformation import (
+    STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
+    ReasoningFieldValues,
+)
 from litellm.llms.fireworks_ai.common_utils import (
     FIREWORKS_FORWARD_USER_ID_PARAM,
     get_fireworks_forwarded_user_id,
@@ -129,3 +133,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
     def supports_native_file_search(self) -> bool:
         return False
+
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
+
+    def ignores_unsupported_reasoning_fields(self) -> bool:
+        return True

litellm/llms/gemini/realtime/transformation.py

@@ -7,6 +7,7 @@ from collections import OrderedDict
 from collections.abc import Mapping, Sequence
 from typing import Final, cast
 
+from pydantic import TypeAdapter
 from typing_extensions import ReadOnly, Required, TypedDict
 
 import litellm
@@ -15,8 +16,9 @@ from litellm._uuid import uuid
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import join_api_base
-from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
+from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
 from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
+    OPENAI_TO_GEMINI_TTS_VOICES,
     VertexGeminiConfig,
 )
 from litellm.responses.litellm_completion_transformation.transformation import (
@@ -76,27 +78,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
 
 # Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped.
 _KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
+_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"})
 
 
 OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
     {"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
 )
 
+# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28)
+GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset(
+    {
+        "achernar",
+        "achird",
+        "algenib",
+        "algieba",
+        "alnilam",
+        "aoede",
+        "autonoe",
+        "callirrhoe",
+        "charon",
+        "despina",
+        "enceladus",
+        "erinome",
+        "fenrir",
+        "gacrux",
+        "iapetus",
+        "kore",
+        "laomedeia",
+        "leda",
+        "orus",
+        "puck",
+        "pulcherrima",
+        "rasalgethi",
+        "sadachbia",
+        "sadaltager",
+        "schedar",
+        "sulafat",
+        "umbriel",
+        "vindemiatrix",
+        "zephyr",
+        "zubenelgenubi",
+    }
+)
 
-def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
+
+def _is_undocumented_live_voice(voice: object) -> bool:
+    return (
+        isinstance(voice, str)
+        and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
+        and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
+    )
+
+
+def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None:
     """Build the Gemini Live speechConfig for a client-requested voice.
 
-    OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
-    the session on an unknown voice, so they are dropped with a warning and
-    the model keeps its default voice. Every other name is forwarded verbatim.
+    OpenAI stock voice names have no Gemini equivalent, so they are always
+    dropped, and so is any name outside Google's documented Live voices when
+    ``drop_undocumented`` is set, keeping the model's default voice. Every
+    other name is forwarded verbatim.
     """
-    if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
+    if not isinstance(voice, str):
+        return VertexGeminiConfig().map_audio_params({"voice": voice})
+    if voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
         verbose_logger.warning(
             "Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
             "dropping it so the session keeps the model's default voice.",
             voice,
         )
         return None
+    if drop_undocumented and _is_undocumented_live_voice(voice):
+        verbose_logger.warning(
+            "Gemini Realtime: voice %s is not a documented Gemini Live voice; "
+            "drop_params is set, so it is dropped and the session keeps the model's default voice.",
+            voice,
+        )
+        return None
     return VertexGeminiConfig().map_audio_params({"voice": voice})
 
 
@@ -142,16 +199,34 @@ def _base64_decoded_byte_count(data: str) -> int:
     return max(len(data) * 3 // 4 - padding, 0)
 
 
+_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
+
+
+def _chat_completion_tool(tool: object) -> dict[str, object]:
+    fields: Final = as_str_mapping(tool) or {}
+    if fields.get("type") != "function" or "function" in fields:
+        return {**fields}
+    return {"type": "function", "function": {key: value for key, value in fields.items() if key != "type"}}
+
+
+def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None:
+    parts: Final = system_instruction.get("parts", []) if system_instruction else []
+    texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str))
+    return "\n".join(texts) if texts else None
+
+
 class GeminiRealtimeConfig(BaseRealtimeConfig):
     _TOOL_CALL_ID_TO_NAME_MAX = 256  # LRU cap for call_id→name mapping
 
-    def __init__(self):
+    def __init__(self, drop_params: bool = False):
         super().__init__()
+        self.drop_params = drop_params
         self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict()
         # Gemini Live sometimes emits usageMetadata in a standalone frame between
         # turns; buffer it here so the next response.done carries the token counts.
         self._pending_usage_metadata: dict | None = None
         self._unbilled_input_audio_bytes: int = 0
+        self._manual_activity_open: bool = False
 
     def is_setup_message(self, msg_obj: dict) -> bool:
         return "setup" in msg_obj
@@ -167,6 +242,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         """Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
         return True
 
+    def _rejects_undocumented_voices(self, model: str) -> bool:
+        """Google AI Studio closes the socket (1007) on an unknown voice only for its native-audio Live models."""
+        return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
+
     @staticmethod
     def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]:
         detail_mapping: Final = as_str_mapping(details)
@@ -272,17 +351,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         except (json.JSONDecodeError, TypeError, AttributeError):
             return False
 
+    def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]:
+        if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request):
+            return ()
+        self._manual_activity_open = True
+        activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}}
+        return (json.dumps({"realtimeInput": activity_start}),)
+
+    def _end_manual_activity(self) -> tuple[str, ...]:
+        if not self._manual_activity_open:
+            return ()
+        self._manual_activity_open = False
+        activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}}
+        return (json.dumps({"realtimeInput": activity_end}),)
+
     def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
         """Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
         if self._manual_turn_detection_enabled(session_configuration_request):
-            realtime_input_dict: BidiGenerateContentRealtimeInput = {
-                "activityEnd": True,
-            }
-            verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
-        else:
-            realtime_input_dict = {"audioStreamEnd": True}
-            verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend")
-        return [json.dumps({"realtimeInput": realtime_input_dict})]
+            return list(self._end_manual_activity())
+        audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True}
+        return [json.dumps({"realtimeInput": audio_stream_end})]
 
     def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection:
         """Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``.
@@ -325,7 +413,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
             "voice",
         ]
 
-    def map_openai_params(self, optional_params: dict, non_default_params: dict) -> dict:
+    def map_openai_params(self, optional_params: dict, non_default_params: dict, *, model: str) -> dict:
         if "generationConfig" not in optional_params:
             optional_params["generationConfig"] = {}
         for key, value in non_default_params.items():
@@ -333,12 +421,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                 optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}])
             elif key == "temperature":
                 optional_params["generationConfig"]["temperature"] = value
-            elif key == "max_response_output_tokens":
+            elif key == "max_response_output_tokens" and isinstance(value, int):
                 optional_params["generationConfig"]["maxOutputTokens"] = value
             elif key == "modalities":
-                optional_params["generationConfig"]["responseModalities"] = [
-                    modality.upper() for modality in cast(list[str], value)
-                ]
+                optional_params["generationConfig"]["responseModalities"] = list(
+                    GeminiRealtimeConfig._requested_response_modalities(cast(object, value))
+                )
             elif key == "tools":
                 from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
                     VertexGeminiConfig,
@@ -347,34 +435,43 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                 vertex_gemini_config = VertexGeminiConfig()
                 # Tools should be at the top level of setup, not inside generationConfig
                 optional_params["tools"] = vertex_gemini_config.map_function(
-                    value=value, optional_params=optional_params
+                    value=[_chat_completion_tool(tool) for tool in _JSON_ARRAY.validate_python(value)],
+                    optional_params=optional_params,
                 )
             elif key == "input_audio_transcription" and value is not None:
                 optional_params["inputAudioTranscription"] = {}
-            elif key == "turn_detection" and value is not None:
-                value_typed = cast(OpenAIRealtimeTurnDetection, value)
-                if (
-                    isinstance(value_typed, dict)
-                    and value_typed.get("type") == "semantic_vad"
-                    and "create_response" not in value_typed
-                ):
-                    # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
-                    # Only skip when there is no create_response override so that
-                    # a guardrail-injected create_response:false is not dropped.
-                    continue
-                transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed)
-                if transformed_audio_activity_config:
-                    optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig(
-                        automaticActivityDetection=transformed_audio_activity_config
-                    )
+            elif key == "turn_detection":
+                realtime_input_config = self._realtime_input_config(cast(object, value))
+                if realtime_input_config is not None:
+                    optional_params["realtimeInputConfig"] = realtime_input_config
             elif key == "voice":
-                speech_config = _gemini_live_speech_config(value)
+                speech_config = _gemini_live_speech_config(
+                    value, self.drop_params and self._rejects_undocumented_voices(model)
+                )
                 if speech_config:
                     optional_params["generationConfig"]["speechConfig"] = speech_config
         if len(optional_params["generationConfig"]) == 0:
             optional_params.pop("generationConfig")
         return optional_params
 
+    def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None:
+        if turn_detection is None:
+            return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True})
+        value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection)
+        if (
+            isinstance(value_typed, dict)
+            and value_typed.get("type") == "semantic_vad"
+            and "create_response" not in value_typed
+        ):
+            # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
+            # Only skip when there is no create_response override so that
+            # a guardrail-injected create_response:false is not dropped.
+            return None
+        transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed)
+        if not transformed_audio_activity_config:
+            return None
+        return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config)
+
     @staticmethod
     def _extract_turn_detection(session: dict) -> dict | None:
         """Extract turn_detection from a session.update payload.
@@ -423,6 +520,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
             if isinstance(input_cfg, dict):
                 if "input_audio_transcription" not in normalized and "transcription" in input_cfg:
                     normalized["input_audio_transcription"] = input_cfg["transcription"]
+                if "turn_detection" not in normalized and "turn_detection" in input_cfg:
+                    normalized["turn_detection"] = input_cfg["turn_detection"]
             output_cfg: Final = audio.get("output")
             if isinstance(output_cfg, dict) and output_cfg.get("voice"):
                 normalized["voice"] = output_cfg["voice"]
@@ -434,7 +533,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         return normalized
 
     @staticmethod
-    def _model_cost_entry(model: str) -> dict:
+    def _model_cost_entry(model: str) -> Mapping[str, object]:
         entry = litellm.model_cost.get(model)
         if entry is None:
             stripped: Final = model.split("/", 1)[-1]
@@ -455,18 +554,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
 
     @staticmethod
-    def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
-        """Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
-        audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
-        normalized: Final = tuple(
-            modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
+    def _requested_response_modalities(value: object) -> tuple[str, ...]:
+        if isinstance(value, str):
+            return (value.upper(),)
+        if not isinstance(value, (list, tuple)):
+            return ()
+        return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value))
+
+    @staticmethod
+    def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]:
+        unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES)
+        if unknown:
+            return unknown
+        if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)):
+            return ("TEXT",)
+        if GeminiRealtimeConfig._is_text_only_live_model(model) and frozenset(requested) == frozenset(("AUDIO",)):
+            return ("AUDIO",)
+        return ()
+
+    def setup_session_update_rejection(
+        self, model: str, session: Mapping[str, object], *, setup_sent: bool
+    ) -> RealtimeSessionUpdateRejection | None:
+        if self.drop_params:
+            return None
+        if setup_sent:
+            return RealtimeSessionUpdateRejection(
+                code="session_update_not_applied",
+                message=(
+                    "Gemini Live accepts one session configuration per connection, and this session already sent "
+                    "its configuration, so this session.update was not applied. Put every setting in session.update "
+                    "before the first conversation content, or set drop_params to drop it instead"
+                ),
+                param="session",
+            )
+        return self._modality_rejection(model, session) or self._voice_rejection(model, session)
+
+    @staticmethod
+    def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
+        param: Final = "output_modalities" if "output_modalities" in session else "modalities"
+        unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities(
+            model, GeminiRealtimeConfig._requested_response_modalities(session.get(param))
         )
+        if not unsupported:
+            return None
+        return RealtimeSessionUpdateRejection(
+            code="invalid_value",
+            message=(
+                f"Invalid value for 'session.{param}': {model} cannot produce "
+                f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead"
+            ),
+            param=f"session.{param}",
+        )
+
+    def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
+        audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {}
+        output_voice: Final = audio_output.get("voice")
+        voice: Final = output_voice or session.get("voice")
+        if not (self._rejects_undocumented_voices(model) and _is_undocumented_live_voice(voice)):
+            return None
+        param: Final = "session.audio.output.voice" if output_voice else "session.voice"
+        return RealtimeSessionUpdateRejection(
+            code="invalid_value",
+            message=(
+                f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. "
+                "Set drop_params to drop it instead"
+            ),
+            param=param,
+        )
+
+    @staticmethod
+    def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
+        """Drop modalities Gemini Live cannot produce at all, swap the ones this
+        model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for
+        text-only ones such as transcribe-live), then keep a single modality
+        because Gemini Live rejects a setup naming two."""
+        return GeminiRealtimeConfig._single_response_modality(
+            GeminiRealtimeConfig._swap_unsupported_response_modalities(
+                model, GeminiRealtimeConfig._known_response_modalities(model, modalities)
+            )
+        )
+
+    @staticmethod
+    def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
+        normalized: Final = tuple(str(modality).upper() for modality in modalities)
+        known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES)
+        if len(known) < len(normalized):
+            verbose_logger.warning(
+                "Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s",
+                model,
+                ", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES),
+            )
+        return known or (GeminiRealtimeConfig._default_response_modality(model),)
+
+    @staticmethod
+    def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]:
         if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
+            verbose_logger.warning(
+                "Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO",
+                model,
+            )
             return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
         if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
             return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
         return normalized
 
+    @staticmethod
+    def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]:
+        if len(modalities) <= 1:
+            return modalities
+        chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0]
+        verbose_logger.warning(
+            "Gemini Live: at most one response modality is allowed per session; using %s for the requested %s",
+            chosen,
+            ", ".join(modalities),
+        )
+        return (chosen,)
+
     @staticmethod
     def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]:
         generation_config: Final = setup.get("generationConfig")
@@ -504,7 +707,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         # would be silently dropped because ``map_openai_params`` only
         # recognises the flat OpenAI-beta key names.
         session_payload = self._normalize_session_payload_for_mapping(session_payload)
-        new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload)
+        new_overrides: Final = self.map_openai_params(
+            optional_params={}, non_default_params=session_payload, model=model
+        )
 
         if session_configuration_request is None:
             generation_config: Final = new_overrides.setdefault("generationConfig", {})
@@ -618,14 +823,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                 message_str = str(message)
             raise ValueError(f"Invalid JSON message: {message_str}")
 
-        messages: Final[list[str]] = []
         msg_type: Final = json_message.get("type")
 
         if msg_type == "session.update":
             return self._handle_session_update(json_message, model, session_configuration_request)
 
         if msg_type == "response.create":
-            return []  # Gemini responds automatically; nothing to forward
+            return list(self._end_manual_activity())
 
         if msg_type == "conversation.item.create":
             return self._handle_conversation_item(json_message)
@@ -643,8 +847,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
 
             gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict})
             verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend")
-            messages.append(gemini_msg)
-            return messages
+            return [*self._start_manual_activity(session_configuration_request), gemini_msg]
 
         if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"):
             return self._handle_input_audio_buffer_commit_or_end(session_configuration_request)
@@ -660,22 +863,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         logging_session_id: str,
         session_configuration_request: str | None = None,
     ) -> OpenAIRealtimeStreamSessionEvents:
-        if session_configuration_request:
-            session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
-        else:
-            session_configuration_request_dict = {}
+        session_configuration_request_dict: Final = _parse_setup(
+            session_configuration_request or self.session_configuration_request(model)
+        )
 
         _model: Final = session_configuration_request_dict.get("model") or model
         generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {}
         gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"])
         _modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)]
-        _system_instruction: Final = session_configuration_request_dict.get("systemInstruction")
+        instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction"))
         session: Final = OpenAIRealtimeStreamSession(
             id=logging_session_id,
             modalities=_modalities,
         )
-        if _system_instruction is not None and isinstance(_system_instruction, str):
-            session["instructions"] = _system_instruction
+        if instructions is not None:
+            session["instructions"] = instructions
         if _model is not None and isinstance(_model, str):
             # Strip Vertex/AI Studio path prefixes to expose the bare model name.
             if "/models/" in _model:
@@ -1662,6 +1864,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         # Deferred setup opt-in: litellm.gemini_live_defer_setup = True
         return not litellm.gemini_live_defer_setup
 
+    def builds_setup_from_session_update(self) -> bool:
+        return True
+
     def session_configuration_request(self, model: str) -> str:
         """
 

litellm/llms/github_copilot/responses/transformation.py

@@ -24,6 +24,7 @@ from litellm._logging import verbose_logger
 from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
 from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.types.llms.openai import (
     ResponseInputParam,
@@ -437,6 +438,12 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
         return False
 
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return STANDARD_MODE_REASONING_FIELD_VALUES
+
+    def ignores_unsupported_reasoning_fields(self) -> bool:
+        return True
+
     def supports_native_websocket(self) -> bool:
         """GitHub Copilot does not support native WebSocket for Responses API"""
         return False

litellm/llms/litellm_proxy/responses/transformation.py

@@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError
 
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.responses.transformation import ANY_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.utils import LlmProviders
@@ -42,6 +43,9 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def custom_llm_provider(self) -> LlmProviders:
         return LlmProviders.LITELLM_PROXY
 
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return ANY_REASONING_FIELD_VALUES
+
     def get_complete_url(
         self,
         api_base: str | None,

litellm/llms/perplexity/responses/transformation.py

@@ -9,6 +9,7 @@ The only provider quirks:
 Ref: https://docs.perplexity.ai/api-reference/responses-post
 """
 
+from types import MappingProxyType
 from typing import Any, Final
 
 import httpx
@@ -17,6 +18,7 @@ from pydantic import TypeAdapter
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.responses.transformation import STANDARD_MODE_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse
@@ -24,6 +26,9 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
 
 _DECODED_JSON: Final = TypeAdapter(object)
+PERPLEXITY_REASONING_FIELD_VALUES: Final[ReasoningFieldValues] = MappingProxyType(
+    {**STANDARD_MODE_REASONING_FIELD_VALUES, "context": frozenset({"auto", "current_turn", "all_turns"})}
+)
 
 
 class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
@@ -127,3 +132,9 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
     def supports_native_websocket(self) -> bool:
         """Perplexity does not support native WebSocket for Responses API"""
         return False
+
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return PERPLEXITY_REASONING_FIELD_VALUES
+
+    def ignores_unsupported_reasoning_fields(self) -> bool:
+        return True

litellm/llms/sagemaker/chat/transformation.py

@@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import LlmProviders
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
-from ..common_utils import AWSEventStreamDecoder, SagemakerError
+from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -59,7 +59,12 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
         inference_component_name: Final = optional_params.get("model_id")
         if not isinstance(inference_component_name, str):
             return headers
-        return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name}
+        other_headers: Final = {
+            name: value
+            for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
+            if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower()
+        }
+        return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name}
 
     def transform_request(
         self,
@@ -76,9 +81,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
             litellm_params=litellm_params,
             headers=headers,
         )
-        container_request: Final = {
-            key: value for key, value in request.items() if key not in self.aws_authentication_params
-        }
+        header_only_params: Final = frozenset((*self.aws_authentication_params, "model_id"))
+        container_request: Final = {key: value for key, value in request.items() if key not in header_only_params}
         served_model_name: Final = litellm_params.get("hf_model_name")
         if not isinstance(served_model_name, str):
             return container_request

litellm/llms/sagemaker/common_utils.py

@@ -14,9 +14,10 @@ from typing_extensions import ReadOnly, Required, TypedDict
 from litellm import verbose_logger
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.utils import GenericStreamingChunk as GChunk
-from litellm.types.utils import StreamingChatCompletionChunk
+from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk
 
 STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
 
 
 @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
@@ -167,6 +168,36 @@ def get_sagemaker_response_stream_shape():
     return _load_sagemaker_response_stream_shape()
 
 
+INFERENCE_COMPONENT_HEADER: Final = "X-Amzn-SageMaker-Inference-Component"
+
+
+_SAGEMAKER_PROVIDERS: Final = frozenset(
+    provider.value for provider in (LlmProviders.SAGEMAKER, LlmProviders.SAGEMAKER_CHAT, LlmProviders.SAGEMAKER_NOVA)
+)
+
+
+def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None:
+    custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
+    if isinstance(custom_llm_provider, str) and custom_llm_provider:
+        return custom_llm_provider
+    model: Final = litellm_params.get("model")
+    return model.split("/", 1)[0] if isinstance(model, str) else None
+
+
+def deployment_inference_component(litellm_params: Mapping[str, object]) -> str | None:
+    headers: Final = litellm_params.get("extra_headers")
+    if _deployment_provider(litellm_params) not in _SAGEMAKER_PROVIDERS or not isinstance(headers, Mapping):
+        return None
+    return next(
+        (
+            value
+            for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
+            if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str)
+        ),
+        None,
+    )
+
+
 def with_inference_component_hint(error_message: str) -> str:
     if "Inference Component Name header is required" in error_message:
         return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`"

litellm/llms/sagemaker/completion/transformation.py

@@ -23,7 +23,7 @@ from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import ModelResponse, Usage
 from litellm.utils import token_counter
 
-from ..common_utils import SagemakerError
+from ..common_utils import STR_KEYED_MAPPING, SagemakerError
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -153,7 +153,9 @@ class SagemakerConfig(BaseConfig):
         litellm_params: dict,
         headers: dict,
     ) -> dict:
-        inference_params: Final = optional_params.copy()
+        inference_params: Final = {
+            key: value for key, value in STR_KEYED_MAPPING.validate_python(optional_params).items() if key != "model_id"
+        }
         stream: Final = inference_params.pop("stream", False)
         data: Final[dict] = {"parameters": inference_params}
         if stream is True:

litellm/llms/subscription_accounts.py

@@ -0,0 +1,37 @@
+"""
+Providers whose requests are covered by a flat subscription, each with the way to read
+the account id of the login a deployment is signed into.
+"""
+
+from collections.abc import Callable, Mapping
+from types import MappingProxyType
+from typing import Final
+
+from litellm.llms.chatgpt.authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
+from litellm.secret_managers.main import UnresolvableSecretReference, resolve_local_oidc_reference
+from litellm.utils import with_named_credential
+
+
+def _chatgpt_account_id(litellm_params: Mapping[str, object]) -> str | None:
+    params: Final = with_named_credential(litellm_params)
+    auth_file: Final = get_chatgpt_auth_file(params)
+    if auth_file:
+        return get_cached_authenticator(auth_file).read_account_id()
+    api_key: Final = params.get("api_key")
+    if not isinstance(api_key, str) or not api_key:
+        return Authenticator().read_account_id()
+    match resolve_local_oidc_reference(api_key):
+        case UnresolvableSecretReference():
+            return None
+        case str() as access_token:
+            return Authenticator().read_account_id(access_token)
+
+
+SUBSCRIPTION_ACCOUNT_ID_RESOLVERS: Final[Mapping[str, Callable[[Mapping[str, object]], str | None]]] = MappingProxyType(
+    {"chatgpt": _chatgpt_account_id}
+)
+
+
+def resolve_subscription_account_id(custom_llm_provider: str, litellm_params: Mapping[str, object]) -> str | None:
+    resolver: Final = SUBSCRIPTION_ACCOUNT_ID_RESOLVERS.get(custom_llm_provider)
+    return None if resolver is None else resolver(litellm_params)

litellm/llms/vertex_ai/realtime/transformation.py

@@ -39,8 +39,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
     (they require async I/O) and injected at construction time.
     """
 
-    def __init__(self, access_token: str, project: str, location: str) -> None:
-        super().__init__()
+    def __init__(self, access_token: str, project: str, location: str, drop_params: bool = False) -> None:
+        super().__init__(drop_params=drop_params)
         self._access_token = access_token
         self._project = project
         self._location = location
@@ -51,6 +51,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
     def _rejects_vad_tuning_with_detection_off(self) -> bool:
         return False
 
+    def _rejects_undocumented_voices(self, model: str) -> bool:
+        return True
+
     # ------------------------------------------------------------------
     # URL
     # ------------------------------------------------------------------
@@ -163,7 +166,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
         # settings would be silently dropped because ``map_openai_params`` only
         # recognises the flat OpenAI-beta key names.
         session_params = self._normalize_session_payload_for_mapping(session_params)
-        setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params)
+        setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params, model=model)
 
         # Use full Vertex AI model path
         setup_config["model"] = self._vertex_model_path(model)
@@ -177,15 +180,16 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
         # ``silence_duration_ms``). ``map_automatic_turn_detection`` sets
         # ``disabled=True`` whenever ``create_response`` is absent or
         # ``False``. Force ``disabled=False`` only when the client did
-        # not explicitly request ``create_response: False`` — that path
+        # not explicitly request ``create_response: False`` or
+        # ``turn_detection: null`` — the first of those paths
         # is how transcription guardrails suppress automatic responses,
         # and overriding it here would silently bypass the guardrail.
         # Vertex Live has no "VAD on, no auto-response" mode, so callers
         # that need that behaviour must accept that VAD is off.
         client_turn_detection: Final = session_params.get("turn_detection")
         client_disabled_auto_response: Final = (
-            isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False
-        )
+            "turn_detection" in session_params and client_turn_detection is None
+        ) or (isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False)
         realtime_input_config: Final = setup_config.setdefault("realtimeInputConfig", {})
         automatic_detection: Final = realtime_input_config.setdefault("automaticActivityDetection", {})
         if not client_disabled_auto_response:
@@ -261,4 +265,5 @@ def vertex_realtime_config(
         access_token=access_token,
         project=project,
         location=VertexBase.get_vertex_region(vertex_region=location, model=model),
+        drop_params=drop_params,
     )

litellm/llms/xai/responses/transformation.py

@@ -10,6 +10,10 @@ from litellm._logging import verbose_logger
 from litellm.constants import XAI_API_BASE
 from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.base_llm.responses.transformation import (
+    STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES,
+    ReasoningFieldValues,
+)
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd
 from litellm.secret_managers.main import get_secret_str
@@ -290,3 +294,9 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def supports_native_websocket(self) -> bool:
         """XAI does not support native WebSocket for Responses API"""
         return False
+
+    def reasoning_field_values(self) -> ReasoningFieldValues:
+        return STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES
+
+    def ignores_unsupported_reasoning_fields(self) -> bool:
+        return True

litellm/main.py

@@ -681,7 +681,7 @@ async def acompletion(
         - The `completion` function is called using `run_in_executor` to execute synchronously in the event loop.
         - If `stream` is True, the function returns an async generator that yields completion lines.
     """
-    fallbacks = kwargs.get("fallbacks", None)
+    fallbacks: Final = kwargs.get("fallbacks", None) or litellm.model_fallbacks
     mock_timeout = kwargs.get("mock_timeout", None)
 
     if mock_timeout is True:
@@ -697,6 +697,7 @@ async def acompletion(
     loop: Final = asyncio.get_event_loop()
     custom_llm_provider = kwargs.get("custom_llm_provider", None)
     _ = _resolve_control_options(kwargs, model)
+    caller_kwargs: Final = cast("Mapping[str, object]", {**kwargs})  # cast-ok: acompletion takes untyped kwargs
 
     ## PROMPT MANAGEMENT HOOKS ##
     #########################################################
@@ -715,15 +716,34 @@ async def acompletion(
         custom_llm_provider=cast(str | None, custom_llm_provider),  # cast-ok: read from untyped kwargs
         tools=tools,
         enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")),  # cast-ok: untyped kwargs
-        api_base=kwargs.get("api_base") or base_url,
+        api_base=base_url or kwargs.get("api_base"),
     )
 
-    if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
-        litellm_logging_obj.should_run_prompt_management_hooks(
-            prompt_id=kwargs.get("prompt_id", None),
+    prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None))  # cast-ok: read from untyped kwargs
+    prompt_management_logger: Final = (
+        litellm_logging_obj.get_custom_logger_for_prompt_management(
+            model=model,
+            non_default_params=kwargs,
+            tools=tools,
+            prompt_id=prompt_id,
+            dynamic_callback_params=litellm_logging_obj.standard_callback_dynamic_params,
+        )
+        if isinstance(litellm_logging_obj, LiteLLMLoggingObj)
+        and litellm_logging_obj.should_run_prompt_management_hooks(
+            prompt_id=prompt_id,
             non_default_params=kwargs,
             tools=tools,
         )
+        else None
+    )
+    each_fallback_writes_its_own_cache_dialect: Final = fallbacks is not None and isinstance(
+        prompt_management_logger, AnthropicCacheControlHook
+    )
+
+    if (
+        isinstance(litellm_logging_obj, LiteLLMLoggingObj)
+        and prompt_management_logger is not None
+        and not each_fallback_writes_its_own_cache_dialect
     ):
         (
             model,
@@ -733,8 +753,9 @@ async def acompletion(
             model=model,
             messages=messages,
             non_default_params=kwargs,
-            prompt_id=kwargs.get("prompt_id", None),
+            prompt_id=prompt_id,
             prompt_variables=kwargs.get("prompt_variables", None),
+            prompt_management_logger=prompt_management_logger,
             tools=tools,
             prompt_label=kwargs.get("prompt_label", None),
             prompt_version=kwargs.get("prompt_version", None),
@@ -826,9 +847,10 @@ async def acompletion(
             litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
         )
 
-    fallbacks = fallbacks or litellm.model_fallbacks
     if fallbacks is not None:
-        response = await async_completion_with_fallbacks(**completion_kwargs, kwargs={"fallbacks": fallbacks, **kwargs})
+        response = await async_completion_with_fallbacks(
+            **completion_kwargs, kwargs={"fallbacks": fallbacks, **caller_kwargs}
+        )
         if response is None:
             raise Exception(
                 "No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details."
@@ -5765,7 +5787,7 @@ def completion(
         custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")),  # cast-ok: untyped kwargs
         tools=tools,
         enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")),  # cast-ok: untyped kwargs
-        api_base=kwargs.get("api_base") or base_url,
+        api_base=base_url or kwargs.get("api_base"),
     )
 
     if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
@@ -6267,6 +6289,9 @@ def completion(
         # reasoningSummary/reasoning_summary without tools (AI SDK) that the first
         # (early) check doesn't cover.
         _reasoning_summary_for_bridge: Final = peek_reasoning_summary_aliases(optional_params)
+        typed_deployment_model_info: Final = str_keyed_mapping(
+            cast(object, model_info)  # cast-ok: model_info is the untyped dict kwarg
+        )
         if responses_api_model_info.get("mode") != "responses":
             responses_api_model_info, model = responses_api_bridge_check(
                 model=model,
@@ -6276,9 +6301,7 @@ def completion(
                 reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")),
                 reasoning_summary=_reasoning_summary_for_bridge,
                 api_base=api_base,
-                deployment_model_info=str_keyed_mapping(
-                    cast(object, model_info)  # cast-ok: model_info is the untyped dict kwarg
-                ),
+                deployment_model_info=typed_deployment_model_info,
             )
 
         # Use base_model (the true underlying model) for Azure model-type
@@ -6290,19 +6313,28 @@ def completion(
 
             optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params)
 
-            if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict):
-                from litellm.responses.utils import openai_bridge_reasoning
+            if isinstance(resolved_reasoning_effort, dict):
+                from litellm.responses.utils import (
+                    RefusedReasoningFields,
+                    bridge_reasoning,
+                    raise_refused_reasoning_fields,
+                )
 
-                optional_params["reasoning_effort"] = openai_bridge_reasoning(
+                bridged_reasoning: Final = bridge_reasoning(
                     requested=resolved_reasoning_effort,
                     effort_kept="reasoning_effort" in optional_params,
                     summary_alias=rs_val,
                     model=model,
+                    base_model=base_model if isinstance(base_model, str) else None,
+                    custom_llm_provider=custom_llm_provider,
+                    api_base=api_base,
+                    deployment_model_info=typed_deployment_model_info,
                     drop_params=drop_unsupported_reasoning,
                     additional_drop_params=routing_kwargs.get("additional_drop_params"),
                 )
-            elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort:
-                optional_params["reasoning_effort"] = resolved_reasoning_effort
+                if isinstance(bridged_reasoning, RefusedReasoningFields):
+                    raise_refused_reasoning_fields(bridged_reasoning, model, custom_llm_provider)
+                optional_params["reasoning_effort"] = bridged_reasoning
             elif rs_val is not None:
                 eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort)
                 if isinstance(eff, dict):

litellm/models/__init__.py

@@ -30,6 +30,7 @@ from litellm.models.organization_membership import LiteLLM_OrganizationMembershi
 from litellm.models.project import LiteLLM_ProjectTable
 from litellm.models.skills import LiteLLM_SkillsTable
 from litellm.models.spend_logs import LiteLLM_ErrorLogs, LiteLLM_SpendLogs
+from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
 from litellm.models.tag import LiteLLM_TagTable
 from litellm.models.team import LiteLLM_TeamTable
 from litellm.models.team_membership import LiteLLM_TeamMembership
@@ -59,6 +60,7 @@ __all__ = [
     "LiteLLM_ProxyModelTable",
     "LiteLLM_SkillsTable",
     "LiteLLM_SpendLogs",
+    "LiteLLM_SubscriptionAccountTable",
     "LiteLLM_TagTable",
     "LiteLLM_TeamMemberTable",
     "LiteLLM_TeamMembership",

litellm/models/subscription_account.py

@@ -0,0 +1,28 @@
+"""
+Subscription account table model.
+
+Canonical definition for ``litellm_subscriptionaccounttable``: the operator-entered
+recurring fee of a provider account whose requests are covered by a flat subscription.
+"""
+
+from datetime import datetime
+
+from pydantic import ConfigDict
+
+from litellm.types.llms.base import LiteLLMPydanticObjectBase
+
+
+class LiteLLM_SubscriptionAccountTable(LiteLLMPydanticObjectBase):
+    model_config = ConfigDict(protected_namespaces=(), frozen=True)
+
+    subscription_account_id: str
+    custom_llm_provider: str
+    account_id: str
+    label: str | None = None
+    monthly_fee: float
+    currency: str
+    billing_period_start: str
+    created_at: datetime
+    created_by: str
+    updated_at: datetime
+    updated_by: str

litellm/proxy/_lazy_openapi_snapshot.json

@@ -1648,7 +1648,7 @@
       },
       "/access_group/{access_group}/update": {
         "put": {
-          "description": "Update an access group's model names.\n\nThis will:\n1. Remove the access group from all current deployments\n2. Add the access group to all deployments for the new model_names list\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n  -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n  -H 'Content-Type: application/json' \\\n  -d '{\n    \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n  }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found",
+          "description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n  -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n  -H 'Content-Type: application/json' \\\n  -d '{\n    \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n  }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit holds the group or one of its models for 30 s",
           "operationId": "update_access_group_access_group__access_group__update_put",
           "parameters": [
             {

litellm/proxy/_types.py

@@ -667,6 +667,10 @@ class LiteLLMRoutes(enum.Enum):
             "/team/spend/by_user",
             # gateway request counts (SGR); deployment-wide, admin-only
             "/gateway/daily/activity",
+            "/subscription_accounts/usage",
+            "/subscription_accounts/new",
+            "/subscription_accounts/update",
+            "/subscription_accounts/delete",
             # model
             "/model/new",
             "/model/update",

litellm/proxy/auth/handle_jwt.py

@@ -2199,15 +2199,19 @@ class JWTAuthManager:
         # not mistaken for claimless and left with stale DB memberships the fallback
         # could later attribute. With the flag off, keep the upstream plural-only
         # reconciliation so existing deployments are unchanged.
-        jwt_team_ids: Final = set(
-            jwt_handler.get_all_jwt_team_ids(jwt_valid_token)
-            if jwt_handler.litellm_jwtauth.fallback_to_db_teams
-            else jwt_handler.get_team_ids_from_jwt(jwt_valid_token)
+        claim_team_ids: Final = tuple(
+            dict.fromkeys(
+                jwt_handler.get_all_jwt_team_ids(jwt_valid_token)
+                if jwt_handler.litellm_jwtauth.fallback_to_db_teams
+                else jwt_handler.get_team_ids_from_jwt(jwt_valid_token)
+            )
         )
-        existing_teams: Final = set(user_object.teams or [])
-        teams_to_add: Final = jwt_team_ids - existing_teams
+        jwt_team_ids: Final = frozenset(claim_team_ids)
+        existing_team_order: Final = tuple(dict.fromkeys(user_object.teams or []))
+        existing_teams: Final = frozenset(existing_team_order)
+        teams_to_add: Final = tuple(team_id for team_id in claim_team_ids if team_id not in existing_teams)
         preserve_db_teams_without_claims: Final = jwt_handler.litellm_jwtauth.fallback_to_db_teams and not jwt_team_ids
-        teams_to_remove: Final = set() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids
+        teams_to_remove: Final = frozenset() if preserve_db_teams_without_claims else existing_teams - jwt_team_ids
         if teams_to_add or teams_to_remove:
             from litellm.proxy.management_endpoints.scim.scim_v2 import (
                 patch_team_membership,
@@ -2218,7 +2222,10 @@ class JWTAuthManager:
                 teams_ids_to_add_user_to=list(teams_to_add),
                 teams_ids_to_remove_user_from=list(teams_to_remove),
             )
-            user_object.teams = list(jwt_team_ids)
+            user_object.teams = [
+                *(team_id for team_id in existing_team_order if team_id not in teams_to_remove),
+                *teams_to_add,
+            ]
             if user_api_key_cache is not None:
                 await user_api_key_cache.async_set_cache(
                     key=user_object.user_id,

litellm/proxy/common_utils/config_sync_pubsub.py

@@ -2,9 +2,12 @@ import asyncio
 import json
 import random
 import time
-from collections.abc import Awaitable, Callable
+from collections.abc import AsyncIterator, Awaitable, Callable
 from dataclasses import asdict, dataclass
-from typing import TYPE_CHECKING, Final, Protocol, cast  # noqa: TID251  # untyped prisma/redis boundary needs cast
+from itertools import chain
+from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast  # noqa: TID251  # untyped prisma/redis boundary
+
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_proxy_logger
 from litellm.repositories.prisma_protocols import RowT_co, TableActions
@@ -106,24 +109,69 @@ _pubsub_capable_client: Final = pubsub_capable_client
 @dataclass(frozen=True, slots=True)
 class _ConfigChangeMessage:
     object_type: str
+    model_ids: tuple[str, ...] = ()
 
 
-def _config_change_message_json(object_type: str) -> str:
-    return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type)))
+@dataclass(frozen=True, slots=True)
+class _PubSubEnvelope:
+    data: str | bytes
 
 
-async def publish_config_change(redis_cache: "RedisCache | None", object_type: str) -> None:
+class _WrittenModelRow(BaseModel):
+    model_config = ConfigDict(from_attributes=True, frozen=True)
+
+    model_id: str
+
+
+_CONFIG_CHANGE_MESSAGE: Final = TypeAdapter(_ConfigChangeMessage)
+_PUBSUB_ENVELOPE: Final = TypeAdapter(_PubSubEnvelope)
+_MODEL_TABLE_NAME: Final = "litellm_proxymodeltable"
+_TARGETED_WRITE_ACTION_NAMES: Final[frozenset[str]] = frozenset({"create", "update", "upsert"})
+
+PublishConfigChange: TypeAlias = Callable[[str, tuple[str, ...]], Awaitable[None]]
+
+
+def _config_change_message_json(object_type: str, model_ids: tuple[str, ...]) -> str:
+    return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type, model_ids=model_ids)))
+
+
+def parse_config_change(message: object) -> _ConfigChangeMessage | None:
+    try:
+        payload: Final = (
+            message if isinstance(message, (str, bytes)) else _PUBSUB_ENVELOPE.validate_python(message).data
+        )
+        return _CONFIG_CHANGE_MESSAGE.validate_json(payload)
+    except ValidationError:
+        return None
+
+
+def written_model_ids(object_type: str, action_name: str, result: object) -> tuple[str, ...]:
+    if object_type != _MODEL_TABLE_NAME or action_name not in _TARGETED_WRITE_ACTION_NAMES:
+        return ()
+    try:
+        return (_WrittenModelRow.model_validate(result).model_id,)
+    except ValidationError:
+        return ()
+
+
+async def publish_config_change(
+    redis_cache: "RedisCache | None", object_type: str, model_ids: tuple[str, ...] = ()
+) -> None:
     if redis_cache is None:
         return
     try:
         client: Final = pubsub_capable_client(redis_cache)
-        await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type))
+        await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type, model_ids))
     except Exception as e:  # noqa: BLE001  # best-effort publish; writes must never fail on redis errors
         verbose_proxy_logger.warning("config sync publish for %s failed: %s", object_type, e)
 
 
-async def publish_config_change_for_object_type(object_type: str) -> None:
-    await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type)
+async def publish_config_change_for_object_type(object_type: str, model_ids: tuple[str, ...] = ()) -> None:
+    await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type, model_ids=model_ids)
+
+
+async def publish_model_rows_written(model_ids: tuple[str, ...]) -> None:
+    await publish_config_change_for_object_type(_MODEL_TABLE_NAME, model_ids)
 
 
 async def publish_config_param_change(param_name: str) -> None:
@@ -139,7 +187,7 @@ async def publish_config_param_change(param_name: str) -> None:
 class _PublishOnWriteActions:
     __slots__ = ("_actions", "_object_type", "_publish")
 
-    def __init__(self, actions: object, object_type: str, publish: Callable[[str], Awaitable[None]]) -> None:
+    def __init__(self, actions: object, object_type: str, publish: PublishConfigChange) -> None:
         self._actions = actions
         self._object_type = object_type
         self._publish = publish
@@ -157,7 +205,7 @@ class _PublishOnWriteActions:
             **kwargs: object,  # kwargs-ok: transparent passthrough to untyped prisma action
         ) -> object:
             result: Final = await write_action(*args, **kwargs)
-            await publish(object_type)
+            await publish(object_type, written_model_ids(object_type, name, result))
             return result
 
         return _write_then_publish
@@ -166,7 +214,7 @@ class _PublishOnWriteActions:
 def wrap_table_actions_for_config_sync(
     actions: "TableActions[RowT_co]",
     table_name: str,
-    publish: Callable[[str], Awaitable[None]] = publish_config_change_for_object_type,
+    publish: PublishConfigChange = publish_config_change_for_object_type,
 ) -> "TableActions[RowT_co]":
     if table_name not in _CONFIG_SYNCED_TABLE_NAMES:
         return actions
@@ -174,8 +222,14 @@ def wrap_table_actions_for_config_sync(
     return cast("TableActions[RowT_co]", wrapped)  # cast-ok: dynamic write-through proxy keeps the wrapped row type
 
 
+def _published_model_ids(messages: tuple[object, ...]) -> tuple[str, ...]:
+    changes: Final = (parse_config_change(message) for message in messages)
+    return tuple(dict.fromkeys(chain.from_iterable(change.model_ids for change in changes if change is not None)))
+
+
 class ConfigSyncSubscriber:
     __slots__ = (
+        "_apply_model_ids",
         "_backoff_initial_seconds",
         "_backoff_max_seconds",
         "_debounce_seconds",
@@ -195,6 +249,7 @@ class ConfigSyncSubscriber:
         self,
         redis_cache: "RedisCache",
         resync_callbacks: tuple[Callable[[], Awaitable[None]], ...],
+        apply_model_ids: Callable[[tuple[str, ...]], Awaitable[None]] | None = None,
         debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS,
         jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS,
         min_resync_interval_seconds: float = CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS,
@@ -206,6 +261,7 @@ class ConfigSyncSubscriber:
     ) -> None:
         self._redis_cache = redis_cache
         self._resync_callbacks = resync_callbacks
+        self._apply_model_ids = apply_model_ids
         self._debounce_seconds = debounce_seconds
         self._jitter_max_seconds = jitter_max_seconds
         self._min_resync_interval_seconds = min_resync_interval_seconds
@@ -266,32 +322,64 @@ class ConfigSyncSubscriber:
             message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=_POLL_TIMEOUT_SECONDS)
             if message is None:
                 continue
+            await self._apply(pubsub, message)
             await self._debounce_and_resync(pubsub)
 
-    async def _debounce_and_resync(self, pubsub: _ConfigSyncPubSub) -> None:
-        await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds))
-        await self._wait_for_min_resync_interval()
+    async def _debounce_and_resync(self, pubsub: ConfigSyncPubSub) -> None:
+        await self._apply_until(
+            pubsub, self._monotonic() + self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)
+        )
+        await self._apply_until(pubsub, self._next_resync_at())
         await self._drain_pending(pubsub)
         await self._run_resync_callbacks()
         self._last_resync_at = self._monotonic()
 
-    async def _wait_for_min_resync_interval(self) -> None:
+    def _next_resync_at(self) -> float:
         if self._last_resync_at is None:
-            return
-        seconds_until_next_resync = self._min_resync_interval_seconds - (self._monotonic() - self._last_resync_at)
-        if seconds_until_next_resync <= 0:
-            return
-        verbose_proxy_logger.debug(
-            "config sync resync throttled for %.1fs to cap fleet-wide reload rate",
-            seconds_until_next_resync,
-        )
-        await self._sleep(seconds_until_next_resync)
+            return self._monotonic()
+        next_resync_at: Final = self._last_resync_at + self._min_resync_interval_seconds
+        throttled_for: Final = next_resync_at - self._monotonic()
+        if throttled_for > 0:
+            verbose_proxy_logger.debug(
+                "config sync resync throttled for %.1fs to cap fleet-wide reload rate",
+                throttled_for,
+            )
+        return next_resync_at
+
+    async def _apply_until(self, pubsub: ConfigSyncPubSub, deadline: float) -> None:
+        while True:
+            remaining = deadline - self._monotonic()
+            if remaining <= 0:
+                return
+            message = await pubsub.get_message(
+                ignore_subscribe_messages=True, timeout=min(remaining, _POLL_TIMEOUT_SECONDS)
+            )
+            if message is not None:
+                await self._apply(pubsub, message)
 
     @staticmethod
     async def _drain_pending(pubsub: ConfigSyncPubSub) -> None:
         while await pubsub.get_message(ignore_subscribe_messages=True, timeout=0) is not None:
             pass
 
+    @staticmethod
+    async def _ready_messages(pubsub: ConfigSyncPubSub) -> AsyncIterator[object]:
+        while (message := await pubsub.get_message(ignore_subscribe_messages=True, timeout=0)) is not None:
+            yield message
+
+    async def _apply(self, pubsub: ConfigSyncPubSub, message: object) -> None:
+        apply: Final = self._apply_model_ids
+        if apply is None:
+            return
+        ready: Final = (message, *[ready_message async for ready_message in self._ready_messages(pubsub)])
+        model_ids: Final = _published_model_ids(ready)
+        if not model_ids:
+            return
+        try:
+            await apply(model_ids)
+        except Exception as e:  # noqa: BLE001  # the debounced full resync retries what a targeted apply could not
+            verbose_proxy_logger.warning("config sync targeted apply of %s failed: %s", model_ids, e)
+
     async def _run_resync_callbacks(self) -> None:
         for callback in self._resync_callbacks:
             try:

litellm/proxy/common_utils/registry_read_through.py

@@ -15,12 +15,13 @@ or the first routed request on this replica would pick a tier it has not loaded.
 import asyncio
 import time
 from collections.abc import Awaitable, Callable, Mapping, Sequence
-from typing import TYPE_CHECKING, Final
+from typing import TYPE_CHECKING, Final, Literal, TypeAlias
 
 from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
 
 if TYPE_CHECKING:
@@ -33,8 +34,14 @@ if TYPE_CHECKING:
     )
 
     from litellm.integrations.custom_guardrail import CustomGuardrail
+    from litellm.repositories.prisma_protocols import TableActions
     from litellm.types.agents import AgentResponse
 
+    _ModelTable: TypeAlias = TableActions[prisma_models.LiteLLM_ProxyModelTable]
+    _ModelRows: TypeAlias = Sequence[prisma_models.LiteLLM_ProxyModelTable]
+
+ModelLoadOutcome: TypeAlias = Literal["loaded", "missing", "no_router", "auto_router"]
+
 READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0
 READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0
 READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20
@@ -123,47 +130,114 @@ async def _resync_model_deployments(model_name: str) -> bool:
         return False
     prisma_client: Final = proxy_server.prisma_client
     assert prisma_client is not None
-    table: Final = ModelRepository(prisma_client).table
-    name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
-    id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
-    rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
+    rows: Final = await _find_rows_by_name_or_id(model_name, ModelRepository(prisma_client).table)
     if not rows:
         return False
-    router: Final = proxy_server.llm_router
-    if router is None:
-        await proxy_server.proxy_config.add_deployment(
-            prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
-        )
-        return proxy_server.llm_router is not None
-    if _holds_an_auto_router(rows):
-        await proxy_server.proxy_config.add_deployment(
-            prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
-        )
-        return _model_is_loaded(model_name)
     async with proxy_server.MODEL_RECONCILE_LOCK:
+        outcome: Final = await _install_model_rows(rows, _always)
+    if outcome in ("no_router", "auto_router"):
+        await _reload_every_model()
+    return outcome == "loaded" or _model_is_loaded(model_name)
+
+
+async def resync_model_deployments_by_ids(model_ids: tuple[str, ...]) -> None:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
+    from litellm.repositories.model_repository import ModelRepository
+
+    if not _db_backed_registries_enabled("models"):
+        return
+    prisma_client: Final = proxy_server.prisma_client
+    assert prisma_client is not None
+    writer_table: Final = ModelRepository(WriterPinnedClient(prisma_client.db)).table
+    async with proxy_server.MODEL_RECONCILE_LOCK:
+        rows: Final = await _find_rows_by_ids(model_ids, writer_table)
+        _ = await _install_model_rows(_without_auto_routers(rows), _names_an_unloaded_credential)
+
+
+async def _find_rows_by_name_or_id(model_name: str, table: "_ModelTable") -> "_ModelRows":
+    name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name}
+    id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name}
+    return await table.find_many(where=name_filter) or await table.find_many(where=id_filter)
+
+
+async def _find_rows_by_ids(model_ids: tuple[str, ...], table: "_ModelTable") -> "_ModelRows":
+    from litellm.repositories.chunked_in import find_many_in
+
+    return list(await find_many_in(table, "model_id", model_ids))
+
+
+async def _install_model_rows(
+    rows: "_ModelRows",
+    needs_credentials: Callable[["_ModelRows"], bool],
+) -> ModelLoadOutcome:
+    from litellm.proxy import proxy_server
+
+    router: Final = proxy_server.llm_router
+    if not rows:
+        return "missing"
+    if router is None:
+        return "no_router"
+    if _holds_an_auto_router(rows):
+        return "auto_router"
+    if needs_credentials(rows):
+        prisma_client: Final = proxy_server.prisma_client
+        assert prisma_client is not None
         await proxy_server.proxy_config.get_credentials(prisma_client=prisma_client)
-        proxy_server.proxy_config._add_deployment(db_models=rows)
-        proxy_server.llm_model_list = router.registry().deployments()
+    proxy_server.proxy_config._add_deployment(db_models=rows)
+    proxy_server.llm_model_list = router.registry().deployments()
+    return "loaded"
+
+
+async def _reload_every_model() -> None:
+    from litellm.proxy import proxy_server
+
+    prisma_client: Final = proxy_server.prisma_client
+    assert prisma_client is not None
+    await proxy_server.proxy_config.add_deployment(
+        prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj
+    )
+
+
+def _always(rows: "_ModelRows") -> bool:
     return True
 
 
-def _holds_an_auto_router(rows: Sequence["prisma_models.LiteLLM_ProxyModelTable"]) -> bool:
+def _holds_an_auto_router(rows: "_ModelRows") -> bool:
     return any(_names_an_auto_router(row.litellm_params) for row in rows)
 
 
+def _without_auto_routers(rows: "_ModelRows") -> "_ModelRows":
+    return list(row for row in rows if not _names_an_auto_router(row.litellm_params))
+
+
+def _names_an_unloaded_credential(rows: "_ModelRows") -> bool:
+    return any(_names_an_unloaded_credential_param(row.litellm_params) for row in rows)
+
+
 _STORED_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
 
 
-def _names_an_auto_router(stored_litellm_params: object) -> bool:
+def _decrypted_param(stored_litellm_params: object, key: str) -> str | None:
     try:
         params: Final = _STORED_LITELLM_PARAMS.validate_python(stored_litellm_params)
     except ValidationError:
-        return False
-    stored_model: Final = params.get("model")
-    if not isinstance(stored_model, str):
-        return False
-    model: Final = decrypt_value_helper(value=stored_model, key="model", return_original_value=True)
-    return isinstance(model, str) and model.startswith("auto_router/")
+        return None
+    stored_value: Final = params.get(key)
+    if not isinstance(stored_value, str):
+        return None
+    value: Final = decrypt_value_helper(value=stored_value, key=key, return_original_value=True)
+    return value if isinstance(value, str) else None
+
+
+def _names_an_auto_router(stored_litellm_params: object) -> bool:
+    model: Final = _decrypted_param(stored_litellm_params, "model")
+    return model is not None and model.startswith("auto_router/")
+
+
+def _names_an_unloaded_credential_param(stored_litellm_params: object) -> bool:
+    credential_name: Final = _decrypted_param(stored_litellm_params, "litellm_credential_name")
+    return credential_name is not None and CredentialAccessor.find_credential(credential_name) is None
 
 
 async def _resync_guardrails(guardrail_name: str) -> bool:

litellm/proxy/health_check.py

@@ -32,7 +32,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import (
     classify_strategy_router_model,
     strategy_router_dependencies,
 )
-from litellm.router_utils.common_utils import is_model_capability_field
+from litellm.router_utils.common_utils import deployment_model_id_pin, is_model_capability_field
 from litellm.types.utils import secret_bearing_wif_litellm_params, server_owned_wif_litellm_params
 
 # Provider routing and server-owned federation or OAuth token-exchange fields. Allowed for proxy
@@ -855,6 +855,8 @@ def update_litellm_params_for_health_check(
       `deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model`
       in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept,
       and `modalities` set there without "audio" turns the audio fields off
+    - sends a SageMaker deployment's `X-Amzn-SageMaker-Inference-Component` extra header as `model_id` when the
+      deployment sets none, so legacy `sagemaker/` probes reach that component the way routed requests do
     """
     incoming_model: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model")
     base_model_owner: Final = deployment_model or (incoming_model if isinstance(incoming_model, str) else None)
@@ -935,9 +937,11 @@ def update_litellm_params_for_health_check(
     audio_output_params: Final = _health_check_audio_output_params(
         _MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner
     )
+    model_id_pin: Final = deployment_model_id_pin(probe_params)
+    model_id_params: Final = {} if model_id_pin is None else {"model_id": model_id_pin}
     return {
         key: value
-        for key, value in {**audio_output_params, **probe_params}.items()
+        for key, value in {**audio_output_params, **model_id_params, **probe_params}.items()
         if not is_model_capability_field(key)
     }
 

litellm/proxy/litellm_pre_call_utils.py

@@ -3188,9 +3188,9 @@ def _fallback_lists(
 ) -> tuple[Sequence[object], ...]:
     """Every fallback list the router can route this request through: the three
     router-level lists, the same three keys from the request body (which replace
-    the router's when present, so the union is the safe superset), and the
-    key/team ``router_settings.fallbacks`` the proxy prefers over router-level
-    fallbacks in ``_configured_fallbacks``."""
+    the router's when present, so the union is the safe superset), and the same
+    three keys from key/team ``router_settings``, which the proxy forwards into
+    the request after this discovery runs."""
     lists: Final[list[Sequence[object]]] = []  # mutable-ok: accumulates each fallback list found
     configured: Final = llm_router.settings()
     for router_list in (
@@ -3205,11 +3205,12 @@ def _fallback_lists(
         if isinstance(entries, list):
             lists.append(cast("Sequence[object]", entries))  # cast-ok: request fallback lists hold mixed entry shapes
     for settings in (router_settings, team_router_settings):
-        key_fallbacks: object = settings.get("fallbacks") if settings is not None else None
-        if isinstance(key_fallbacks, list):
-            lists.append(
-                cast("Sequence[object]", key_fallbacks)  # cast-ok: settings fallback lists hold mixed entry shapes
-            )
+        for key in _REQUEST_FALLBACK_KEYS:
+            key_fallbacks: object = settings.get(key) if settings is not None else None
+            if isinstance(key_fallbacks, list):
+                lists.append(
+                    cast("Sequence[object]", key_fallbacks)  # cast-ok: settings fallback lists hold mixed entry shapes
+                )
     return tuple(lists)
 
 

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -12,7 +12,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
 
 from litellm import constants
 from litellm._logging import verbose_proxy_logger
-from litellm.constants import PTU_SENTINEL_API_KEY
+from litellm.constants import PTU_SENTINEL_API_KEY, SUBSCRIPTION_BACKED_PROVIDERS
 from litellm.proxy._types import CommonProxyErrors
 from litellm.proxy.spend_tracking.key_metadata_recovery import (
     attach_user_details,
@@ -295,6 +295,12 @@ def _entity_metadata(
     return stored if stored is not None else {}
 
 
+def provider_bucket_metadata(provider: str, base: Mapping[str, object]) -> Mapping[str, object]:
+    if provider in SUBSCRIPTION_BACKED_PROVIDERS:
+        return {**base, "subscription_covered": True}
+    return dict(base)
+
+
 def update_breakdown_metrics(
     breakdown: BreakdownMetrics,
     record: DailySpendRecord,
@@ -395,7 +401,7 @@ def update_breakdown_metrics(
         if provider not in breakdown.providers:
             breakdown.providers[provider] = MetricWithMetadata(
                 metrics=SpendMetrics(),
-                metadata=provider_metadata.get(provider, {}),  # Add any provider-specific metadata here
+                metadata=provider_bucket_metadata(provider, provider_metadata.get(provider, {})),
             )
         breakdown.providers[provider].metrics = update_metrics(breakdown.providers[provider].metrics, record)
 
@@ -795,12 +801,23 @@ def _aggregate_grouping_sets_records_sync(
             grouped_data[date_str] = bucket
         return bucket
 
-    def assign_metric_with_metadata(target: dict[str, MetricWithMetadata], key: str, metrics: SpendMetrics) -> None:
+    def assign_metric_with_metadata(
+        target: dict[str, MetricWithMetadata],
+        key: str,
+        metrics: SpendMetrics,
+        metadata: Mapping[str, object] | None = None,
+    ) -> None:
         existing: Final = target.get(key)
-        if existing is None:
-            target[key] = MetricWithMetadata(metrics=metrics, metadata={})
-        else:
-            existing.metrics = sum_spend_metrics((existing.metrics, metrics))
+        target[key] = (
+            MetricWithMetadata(metrics=metrics, metadata=dict(metadata or {}))
+            if existing is None
+            else existing.model_copy(
+                update={
+                    "metrics": sum_spend_metrics((existing.metrics, metrics)),
+                    "metadata": {**existing.metadata, **(metadata or {})},
+                }
+            )
+        )
 
     def assign_api_key_breakdown(
         target: dict[str, MetricWithMetadata],
@@ -864,7 +881,9 @@ def _aggregate_grouping_sets_records_sync(
             # provider the base build reported.
             provider_metrics = metrics.model_copy(update={"flat_cost": 0.0})
             provider = record.custom_llm_provider or "unknown"
-            assign_metric_with_metadata(breakdown.providers, provider, provider_metrics)
+            assign_metric_with_metadata(
+                breakdown.providers, provider, provider_metrics, provider_bucket_metadata(provider, {})
+            )
         elif level == _GROUP_DATE_PROVIDER_API_KEY:
             if not is_ptu_sentinel:
                 provider = record.custom_llm_provider or "unknown"

litellm/proxy/management_endpoints/fallback_management_endpoints.py

@@ -14,10 +14,12 @@ import json
 from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Annotated, Final, Literal
 
-from pydantic import Field, JsonValue, TypeAdapter
+from pydantic import Field, JsonValue, TypeAdapter, ValidationError
 
+import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.auth.auth_utils import fallback_target_model_name
 from litellm.proxy.auth.model_checks import get_all_fallbacks
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.utils import PrismaClient, invalidate_config_param
@@ -26,6 +28,7 @@ if TYPE_CHECKING:
     from fastapi import APIRouter, Depends, HTTPException, status
 
     from litellm.proxy.proxy_server import ProxyConfig
+    from litellm.router import Router
 else:
     try:
         from fastapi import APIRouter, Depends, HTTPException, status
@@ -41,6 +44,7 @@ from litellm.types.management_endpoints.router_settings_endpoints import (
     FallbackGetResponse,
     FallbackResponse,
 )
+from litellm.types.router import ContentPolicyFallbackRule, ContentPolicyFallbackRules, FallbackTargets
 
 router: Final = APIRouter()
 
@@ -50,12 +54,83 @@ StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_m
 _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback])
 _JSON_RULES: Final = TypeAdapter(list[JsonValue])
 _STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue])
+_OPTIONAL_JSON_RULES: Final[TypeAdapter[Sequence[JsonValue] | None]] = TypeAdapter(Sequence[JsonValue] | None)
+_CONTENT_POLICY_FALLBACK_RULE: Final[TypeAdapter[ContentPolicyFallbackRule]] = TypeAdapter(ContentPolicyFallbackRule)
+_FALLBACK_TARGETS: Final[TypeAdapter[FallbackTargets]] = TypeAdapter(FallbackTargets)
 
 
 def _rule_covers(entry: StoredFallback, model: str) -> bool:
     return isinstance(entry, dict) and model in entry
 
 
+def _layered_over_config_rules(
+    runtime_rules: ContentPolicyFallbackRules, config_rules: Sequence[Mapping[str, JsonValue]]
+) -> Sequence[Mapping[str, object]]:
+    covered: Final = frozenset[str]().union(*runtime_rules)
+    return [*runtime_rules, *(rule for rule in config_rules if covered.isdisjoint(rule))]
+
+
+def _config_file_rules(rules: object) -> Sequence[Mapping[str, JsonValue]]:
+    try:
+        candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or ()
+    except ValidationError:
+        return ()
+    return tuple(rule for rule in candidates if isinstance(rule, Mapping))
+
+
+def _log_ignored_content_policy_rules(rules: object, error: ValidationError) -> None:
+    verbose_proxy_logger.error(
+        "Ignoring invalid content_policy_fallbacks %r from the DB, every other rule still applies. "
+        "Fix it through POST /fallback or /config/update: %s",
+        rules,
+        error,
+    )
+
+
+def _checked_content_policy_rule(rule: JsonValue) -> ContentPolicyFallbackRule | ValidationError:
+    try:
+        return _CONTENT_POLICY_FALLBACK_RULE.validate_python(rule)
+    except ValidationError as invalid_rule:
+        return invalid_rule
+
+
+def _checked_content_policy_rules(
+    rules: object,
+) -> Sequence[tuple[object, ContentPolicyFallbackRule | ValidationError]]:
+    try:
+        candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or ()
+    except ValidationError as not_a_rule_list:
+        return ((rules, not_a_rule_list),)
+    return tuple((candidate, _checked_content_policy_rule(candidate)) for candidate in candidates)
+
+
+def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None:
+    checked_rules: Final = _checked_content_policy_rules(rules)
+    runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError))
+    layered_rules: Final = (
+        _layered_over_config_rules(
+            runtime_rules,
+            _config_file_rules(
+                litellm.content_policy_fallbacks  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # untyped module list
+            ),
+        )
+        or None
+    )
+    for rule, result in checked_rules:
+        if log_skipped_rules and isinstance(result, ValidationError):
+            _log_ignored_content_policy_rules(rule, result)
+    if layered_rules == llm_router.settings().content_policy_fallbacks:  # pyright: ignore[reportUnknownMemberType]  # untyped RouterSettings list
+        return
+    llm_router.apply(ChangeSet(settings={"content_policy_fallbacks": layered_rules}))
+
+
+def _apply_fallback_rules(llm_router: "Router", fallback_key: str, rules: Sequence[StoredFallback]) -> None:
+    if fallback_key == "content_policy_fallbacks":
+        apply_content_policy_fallbacks(llm_router, rules, log_skipped_rules=True)
+        return
+    llm_router.apply(ChangeSet(assign={fallback_key: rules}))
+
+
 async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None:
     await proxy_config.get_config()
 
@@ -83,6 +158,7 @@ def _editable_rules(
 async def _write_rules(
     prisma_client: PrismaClient,
     proxy_config: "ProxyConfig",
+    llm_router: "Router",
     stored_settings: Mapping[str, JsonValue],
     fallback_key: str,
     rules: Sequence[StoredFallback],
@@ -91,8 +167,10 @@ async def _write_rules(
         section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)}
     )
     if proxy_config.router_settings.owned_by_config(fallback_key):
+        llm_router.apply(ChangeSet(assign={fallback_key: rules}))
         return
     await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules})
+    _apply_fallback_rules(llm_router, fallback_key, rules)
 
 
 async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None:
@@ -218,10 +296,7 @@ async def create_fallback(
             # Add new fallback
             existing_fallbacks.append({data.model: data.fallback_models})
 
-        await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, existing_fallbacks)
-
-        # Update the in-memory router configuration
-        llm_router.apply(ChangeSet(assign={fallback_key: existing_fallbacks}))
+        await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, existing_fallbacks)
 
         verbose_proxy_logger.info(
             "Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type
@@ -277,7 +352,10 @@ async def get_fallback(
             )
 
         # Get fallbacks using the existing utility function
-        fallback_models: Final = get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
+        fallback_targets: Final = _FALLBACK_TARGETS.validate_python(
+            get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
+        )
+        fallback_models: Final = [fallback_target_model_name(target) or model for target in fallback_targets]
 
         if not fallback_models:
             raise HTTPException(
@@ -367,10 +445,7 @@ async def delete_fallback(
                 detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"},
             )
 
-        await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, updated_fallbacks)
-
-        # Update the in-memory router configuration
-        llm_router.apply(ChangeSet(assign={fallback_key: updated_fallbacks}))
+        await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, updated_fallbacks)
 
         verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type)
 

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -13,6 +13,7 @@ These are members of a Team on LiteLLM
 """
 
 import asyncio
+import contextlib
 import json
 import traceback
 from collections.abc import Awaitable, Mapping, Sequence
@@ -443,30 +444,25 @@ async def add_new_user_to_default_team(
     teams: Sequence[str | NewUserRequestTeam],
     prisma_client: "PrismaClient",
 ):
-    tasks: Final[list[Awaitable[object]]] = []
-    for team in teams:
-        user_role: Literal["user", "admin"] = "user"
-        max_budget_in_team: float | None = None
-        if isinstance(team, str):
-            team_id = team
-        elif isinstance(team, NewUserRequestTeam):
-            team_id = team.team_id
-            user_role = team.user_role
-            max_budget_in_team = team.max_budget_in_team
-        else:
-            raise ValueError(f"Invalid team type: {type(team)}")
-
-        tasks.append(
-            _add_user_to_team(
+    memberships: Final = tuple(_default_team_membership(team) for team in teams)
+    for membership in memberships:
+        with contextlib.suppress(Exception):
+            await _add_user_to_team(
                 user_id=user_id,
-                team_id=team_id,
+                team_id=membership.team_id,
                 user_email=user_email,
                 user_api_key_dict=user_api_key_dict,
-                max_budget_in_team=max_budget_in_team,
-                user_role=user_role,
+                max_budget_in_team=membership.max_budget_in_team,
+                user_role=membership.user_role,
             )
-        )
-    await asyncio.gather(*tasks, return_exceptions=True)
+
+
+def _default_team_membership(team: str | NewUserRequestTeam) -> NewUserRequestTeam:
+    if isinstance(team, str):
+        return NewUserRequestTeam(team_id=team)
+    if isinstance(team, NewUserRequestTeam):
+        return team
+    raise ValueError(f"Invalid team type: {type(team)}")
 
 
 async def _fetch_user_team_ids(user_id: str, prisma_client: "PrismaClient") -> tuple[str, ...]:

litellm/proxy/management_endpoints/model_access_group_management_endpoints.py

@@ -12,23 +12,30 @@ Endpoints here:
 - DELETE /access_group/{access_group}/budget - Clear an access group's shared budget
 """
 
+import asyncio
+import contextlib
+import itertools
 import json
-from collections.abc import Mapping, Sequence
-from datetime import datetime
+import time
+from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from datetime import datetime, timedelta
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Annotated, Any, Final, Protocol
 
 from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel, ConfigDict, TypeAdapter
 from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.config_sync_pubsub import publish_model_rows_written
 from litellm.proxy.common_utils.user_api_key_cache import (
     UserApiKeyCache,
     model_access_group_cache_key,
     model_access_group_registry_cache_key,
 )
+from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper
 from litellm.proxy.management_endpoints.common_utils import validate_budget_duration
 
 # Clear cache and reload models to pick up the access group changes
@@ -116,6 +123,28 @@ class _ModelTableClient(Protocol):
     async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
 
 
+class _ModelTableBatchActions(Protocol):
+    def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ...
+
+
+class _ModelTableBatch(Protocol):
+    @property
+    def litellm_proxymodeltable(self) -> _ModelTableBatchActions: ...
+
+
+class _ModelTableTxActions(Protocol):
+    async def find_many(self) -> Sequence[_DeploymentRow]: ...
+
+
+class _ModelTableTx(Protocol):
+    @property
+    def litellm_proxymodeltable(self) -> _ModelTableTxActions: ...
+
+    async def query_raw(self, query: str, *args: object) -> object: ...
+
+    def batch_(self) -> contextlib.AbstractAsyncContextManager[_ModelTableBatch]: ...
+
+
 class _BudgetRow(Protocol):
     @property
     def budget_id(self) -> str: ...
@@ -168,7 +197,42 @@ class _ModelAccessGroupBudgetTableClient(Protocol):
 
 
 def _model_table(prisma_client: PrismaClient) -> _ModelTableClient:
-    return ModelRepository(prisma_client).table
+    return ModelRepository(WriterPinnedClient(prisma_client.db)).table
+
+
+_TRY_LOCK_ACCESS_GROUP: Final = (
+    "SELECT pg_try_advisory_xact_lock(hashtext('model_access_group'), hashtext($1)) AS locked"
+)
+_LOCK_MODEL_ROWS: Final = (
+    'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = ANY($1::text[]) '
+    "ORDER BY model_id FOR UPDATE SKIP LOCKED"
+)
+_MEMBERSHIP_TX_TIMEOUT: Final = timedelta(seconds=30)
+_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds()
+_LOCK_RETRY_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8)
+_LOCK_RETRY_LONGEST_DELAY_SECONDS: Final = 1.0
+
+
+class _LockedModelRow(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model_id: str
+
+
+_LOCKED_MODEL_ROWS: Final = TypeAdapter(tuple[_LockedModelRow, ...])
+
+
+class _AccessGroupLock(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    locked: bool
+
+
+_ACCESS_GROUP_LOCK: Final = TypeAdapter(tuple[_AccessGroupLock])
+
+
+def _model_table_tx(prisma_client: PrismaClient) -> contextlib.AbstractAsyncContextManager[_ModelTableTx]:
+    return writer_wrapper(prisma_client.db).tx(timeout=_MEMBERSHIP_TX_TIMEOUT)
 
 
 def _model_access_group_budget_table(prisma_client: PrismaClient) -> _ModelAccessGroupBudgetTableClient:
@@ -379,11 +443,10 @@ async def _tag_deployment_with_access_group(
     """Write `access_group` into one deployment's model_info; returns the
     (model_id, updated model_info) pair when a write happened, None when the
     deployment already carried the group."""
-    updated_model_info, was_modified = add_access_group_to_deployment(
-        model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
-        access_group=access_group,
+    updated_model_info: Final = _model_info_with_access_group(
+        model_id=model_id, model_info=model_info, access_group=access_group
     )
-    if not was_modified:
+    if updated_model_info is None:
         return None
     await _model_table(prisma_client).update(
         where={"model_id": model_id},
@@ -393,6 +456,141 @@ async def _tag_deployment_with_access_group(
     return (model_id, updated_model_info)
 
 
+def _model_info_with_access_group(model_id: str, model_info: object, access_group: str) -> Mapping[str, object] | None:
+    updated_model_info, was_modified = add_access_group_to_deployment(
+        model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
+        access_group=access_group,
+    )
+    return updated_model_info if was_modified else None
+
+
+def _model_info_without_access_group(
+    model_id: str, model_info: object, access_group: str
+) -> Mapping[str, object] | None:
+    updated_model_info, was_modified = remove_access_group_from_deployment(
+        model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
+        access_group=access_group,
+    )
+    return updated_model_info if was_modified else None
+
+
+def _membership_rewrite(deployment: _DeploymentRow, selected: bool, access_group: str) -> Mapping[str, object] | None:
+    if selected:
+        return _model_info_with_access_group(
+            model_id=deployment.model_id, model_info=deployment.model_info, access_group=access_group
+        )
+    return _model_info_without_access_group(
+        model_id=deployment.model_id, model_info=deployment.model_info, access_group=access_group
+    )
+
+
+def _membership_rewrites(
+    deployments: Sequence[_DeploymentRow], selected_model_ids: frozenset[str], access_group: str
+) -> tuple[tuple[str, Mapping[str, object]], ...]:
+    rewrites: Final = (
+        (
+            deployment.model_id,
+            _membership_rewrite(
+                deployment=deployment,
+                selected=deployment.model_id in selected_model_ids,
+                access_group=access_group,
+            ),
+        )
+        for deployment in deployments
+    )
+    return tuple((model_id, model_info) for model_id, model_info in rewrites if model_info is not None)
+
+
+def _carries_access_group(model_info: object, access_group: str) -> bool:
+    access_groups: Final = (model_info_as_mapping(model_info) or {}).get("access_groups")
+    return isinstance(access_groups, list) and access_group in access_groups
+
+
+def _model_ids_to_lock(
+    deployments: Sequence[_DeploymentRow], access_group: str, data: UpdateModelGroupRequest
+) -> frozenset[str]:
+    selected_model_ids: Final = frozenset(data.model_ids or ())
+    selected_model_names: Final = frozenset(() if data.model_ids else data.model_names or ())
+    return frozenset(
+        deployment.model_id
+        for deployment in deployments
+        if deployment.model_id in selected_model_ids
+        or deployment.model_name in selected_model_names
+        or _carries_access_group(deployment.model_info, access_group)
+    )
+
+
+async def _raise_404_unless_the_primary_carries(access_group: str, prisma_client: PrismaClient) -> None:
+    if access_group not in _access_groups_of_deployments(await _model_table(prisma_client).find_many()):
+        raise _http_error(404, f"Access group '{access_group}' not found")
+
+
+async def _rewrite_membership_unless_locked(
+    prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest
+) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None:
+    async with _model_table_tx(prisma_client) as tx:
+        (group_lock,) = _ACCESS_GROUP_LOCK.validate_python(await tx.query_raw(_TRY_LOCK_ACCESS_GROUP, access_group))
+        if not group_lock.locked:
+            return None
+        model_ids_to_lock: Final = _model_ids_to_lock(
+            deployments=await tx.litellm_proxymodeltable.find_many(), access_group=access_group, data=data
+        )
+        locked_rows: Final = _LOCKED_MODEL_ROWS.validate_python(
+            await tx.query_raw(_LOCK_MODEL_ROWS, sorted(model_ids_to_lock))
+        )
+        if frozenset(row.model_id for row in locked_rows) != model_ids_to_lock:
+            return None
+        deployments: Final = tuple(
+            deployment
+            for deployment in await tx.litellm_proxymodeltable.find_many()
+            if deployment.model_id in model_ids_to_lock
+        )
+        if access_group not in _access_groups_of_deployments(deployments):
+            raise HTTPException(
+                status_code=404,
+                detail={"error": f"Access group '{access_group}' not found"},
+            )
+        selected_deployments: Final = _selected_deployments_or_400(deployments=deployments, data=data)
+        rewrites: Final = _membership_rewrites(
+            deployments=deployments,
+            selected_model_ids=frozenset(deployment.model_id for deployment in selected_deployments),
+            access_group=access_group,
+        )
+        if rewrites:
+            async with tx.batch_() as batch:
+                for model_id, model_info in rewrites:
+                    batch.litellm_proxymodeltable.update(
+                        where={"model_id": model_id}, data={"model_info": json.dumps(model_info)}
+                    )
+    return selected_deployments, rewrites
+
+
+def _lock_retry_delays() -> Iterator[float]:
+    return itertools.chain(_LOCK_RETRY_BACKOFF_SECONDS, itertools.repeat(_LOCK_RETRY_LONGEST_DELAY_SECONDS))
+
+
+async def _rewrite_membership_once_unlocked(
+    prisma_client: PrismaClient,
+    access_group: str,
+    data: UpdateModelGroupRequest,
+    retry_budget_seconds: float = _LOCK_RETRY_BUDGET_SECONDS,
+    sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+    clock: Callable[[], float] = time.monotonic,
+) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]:
+    deadline: Final = clock() + retry_budget_seconds
+    for delay in _lock_retry_delays():
+        if (
+            rewritten := await _rewrite_membership_unless_locked(
+                prisma_client=prisma_client, access_group=access_group, data=data
+            )
+        ) is not None:
+            return rewritten
+        if clock() + delay > deadline:
+            break
+        await sleep(delay)
+    raise _http_error(409, f"Another edit holds access group '{access_group}' or one of its models, try again")
+
+
 def _readable_model_info_or_raise(model_id: str, model_info: object) -> Mapping[str, object]:
     """These helpers rewrite the model_info column wholesale, so a present-but-unreadable
     value must refuse loudly rather than be silently replaced with a fresh object; an
@@ -412,11 +610,10 @@ async def _strip_access_group_from_deployment(
     """Remove `access_group` from one deployment's model_info; returns the
     (model_id, updated model_info) pair when a write happened, None when the
     deployment did not carry the group."""
-    updated_model_info, was_modified = remove_access_group_from_deployment(
-        model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)),
-        access_group=access_group,
+    updated_model_info: Final = _model_info_without_access_group(
+        model_id=model_id, model_info=model_info, access_group=access_group
     )
-    if not was_modified:
+    if updated_model_info is None:
         return None
     await _model_table(prisma_client).update(
         where={"model_id": model_id},
@@ -493,6 +690,35 @@ async def update_specific_deployments_with_access_group(
     return tuple(pair for pair in tagged if pair is not None)
 
 
+def _selected_deployments_or_400(
+    deployments: Sequence[_DeploymentRow], data: UpdateModelGroupRequest
+) -> tuple[_DeploymentRow, ...]:
+    if data.model_ids:
+        known_model_ids: Final = frozenset(deployment.model_id for deployment in deployments)
+        unknown_model_id: Final = next(
+            (model_id for model_id in data.model_ids if model_id not in known_model_ids), None
+        )
+        if unknown_model_id is not None:
+            raise HTTPException(
+                status_code=400,
+                detail={"error": f"Deployment with model_id '{unknown_model_id}' not found in Database."},
+            )
+        selected_model_ids: Final = frozenset(data.model_ids)
+        return tuple(deployment for deployment in deployments if deployment.model_id in selected_model_ids)
+    model_names: Final = data.model_names or ()
+    known_model_names: Final = frozenset(deployment.model_name for deployment in deployments)
+    unknown_model_name: Final = next((name for name in model_names if name not in known_model_names), None)
+    if unknown_model_name is not None:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": f"Can't find model '{unknown_model_name}' in Database. Access group management is only supported for database models."
+            },
+        )
+    selected_model_names: Final = frozenset(model_names)
+    return tuple(deployment for deployment in deployments if deployment.model_name in selected_model_names)
+
+
 async def _find_deployment_or_400(model_id: str, prisma_client: PrismaClient) -> object:
     deployment: Final = await _model_table(prisma_client).find_unique(where={"model_id": model_id})
     if deployment is None:
@@ -536,10 +762,10 @@ async def get_all_access_groups_from_db(
     Returns:
         Dict[str, AccessGroupInfo]: Dictionary mapping access_group name to info
     """
-    # Get all deployments
-    deployments: Final = await ModelRepository(prisma_client).table.find_many()
+    return dict(_access_groups_of_deployments(await ModelRepository(prisma_client).table.find_many()))
 
-    # Build access group map
+
+def _access_groups_of_deployments(deployments: Sequence[_DeploymentRow]) -> Mapping[str, AccessGroupInfo]:
     model_names_by_group: Final[dict[str, list[str]]] = {}
 
     for deployment in deployments:
@@ -835,8 +1061,8 @@ async def update_access_group(
     Update an access group's model names.
     
     This will:
-    1. Remove the access group from all current deployments
-    2. Add the access group to all deployments for the new model_names list
+    1. Add the access group to all deployments for the new model_names list
+    2. Remove the access group from every other deployment
     
     Example:
     ```bash
@@ -858,6 +1084,7 @@ async def update_access_group(
     Raises:
     - HTTPException 400: If any model names don't exist
     - HTTPException 404: If access group not found
+    - HTTPException 409: If another edit holds the group or one of its models for 30 s
     """
     from litellm.proxy.proxy_server import llm_router, prisma_client
 
@@ -879,26 +1106,14 @@ async def update_access_group(
             detail={"error": "Either model_names or model_ids must be provided and non-empty"},
         )
 
-    use_model_ids: Final = has_model_ids
-
-    # Validation: Check if access group exists
     try:
-        access_groups_map: Final = await get_all_access_groups_from_db(prisma_client=prisma_client)
-        if access_group not in access_groups_map:
-            raise HTTPException(
-                status_code=404,
-                detail={"error": f"Access group '{access_group}' not found"},
-            )
+        await _raise_404_unless_the_primary_carries(access_group=access_group, prisma_client=prisma_client)
     except HTTPException:
         raise
     except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail={"error": f"Failed to check access group existence: {e}"},
-        )
+        raise _http_error(500, f"Failed to check access group existence: {e}")
 
-    # Validation: Check if all new models exist (only if using model_names path)
-    if not use_model_ids and has_model_names:
+    if not has_model_ids and has_model_names:
         assert data.model_names is not None
         missing_models: Final = await _missing_models_after_read_through(
             model_names=data.model_names,
@@ -912,43 +1127,19 @@ async def update_access_group(
             )
 
     try:
-        # Step 1: Remove access group from ALL DB deployments (skip config models)
-        all_deployments: Final = await _model_table(prisma_client).find_many()
-
-        stripped: Final = [
-            await _strip_access_group_from_deployment(
-                model_id=deployment.model_id,
-                model_info=deployment.model_info,
-                access_group=access_group,
-                prisma_client=prisma_client,
-            )
-            for deployment in all_deployments
-        ]
-        stripped_pairs: Final = tuple(pair for pair in stripped if pair is not None)
-
-        # Step 2: Add access group using the appropriate method
-        if use_model_ids:
-            assert data.model_ids is not None
-            updated_pairs = await update_specific_deployments_with_access_group(
-                model_ids=data.model_ids,
-                access_group=access_group,
-                prisma_client=prisma_client,
-            )
-        else:
-            assert data.model_names is not None
-            updated_pairs = await update_deployments_with_access_group(
-                model_names=data.model_names,
-                access_group=access_group,
-                prisma_client=prisma_client,
-            )
-        models_updated: Final = len(updated_pairs)
+        selected_deployments, rewrites = await _rewrite_membership_once_unlocked(
+            prisma_client=prisma_client, access_group=access_group, data=data
+        )
+        if rewrites:
+            await publish_model_rows_written(tuple(model_id for model_id, _ in rewrites))
+        models_updated: Final = len(selected_deployments)
 
         # Clear cache and reload models to pick up the access group changes
         live_before_reload: Final = live_model_ids_snapshot()
         reload_outcome: Final = await clear_cache()
         _raise_http_if_reload_degraded_serving(
             before=live_before_reload,
-            written_models=list({**dict(stripped_pairs), **dict(updated_pairs)}.items()),
+            written_models=rewrites,
             access_group=access_group,
             still_desired=reload_outcome.still_desired,
             live_after=reload_outcome.live_after,

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -779,7 +779,11 @@ async def _auto_router_capability_slot(
                 limit=limit,
             )
         yield tables.litellm_proxymodeltable
-    await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable")
+    await publish_config_change(
+        redis_cache=coordination_redis_cache(),
+        object_type="litellm_proxymodeltable",
+        model_ids=() if model_id is None else (model_id,),
+    )
 
 
 ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add"

litellm/proxy/management_endpoints/scim/scim_v2.py

@@ -229,7 +229,11 @@ class UserProvisionerHelpers:
             return None
 
         requested_teams: Final = list(dict.fromkeys(new_user_request.teams or []))
-        new_teams: Final = requested_teams if requested_teams else list(existing_user.teams or [])
+        new_teams: Final = (
+            list(_keep_record_team_order(existing_user.teams or [], requested_teams))
+            if requested_teams
+            else list(existing_user.teams or [])
+        )
 
         if new_user_request.user_id != existing_user.user_id:
             verbose_proxy_logger.info(
@@ -1165,7 +1169,7 @@ async def _handle_team_membership_changes(
     existing_teams_set: Final = set(existing_teams)
     new_teams_set: Final = set(new_teams)
 
-    teams_to_add: Final = new_teams_set - existing_teams_set
+    teams_to_add: Final = tuple(team for team in dict.fromkeys(new_teams) if team not in existing_teams_set)
     teams_to_remove: Final = existing_teams_set - new_teams_set
 
     if teams_to_add or teams_to_remove:
@@ -1948,7 +1952,11 @@ async def update_user(
 
         # SCIM User.groups is readOnly (RFC 7643 4.1.2): IdPs sync membership via /Groups and send
         # no groups or `groups: []` on profile PUTs, so empty means unspecified, not "remove from every team"
-        target_teams: Final = user_data["teams"] or existing_user.teams
+        target_teams: Final = (
+            list(_keep_record_team_order(existing_user.teams, user_data["teams"]))
+            if user_data["teams"]
+            else existing_user.teams
+        )
         await _handle_team_membership_changes(
             user_id=user_id,
             existing_teams=existing_user.teams,
@@ -2241,19 +2249,29 @@ def _handle_name_update(path: str, op_type: str, value: object, scim_metadata: d
             scim_metadata["familyName"] = str(value)
 
 
-def _handle_group_operations(op_type: str, value: object, teams_set: set[str], path: str | None) -> set[str] | None:
-    """Handle group/team membership operations."""
-    group_values = _extract_group_values(value)
-    if not group_values and value is None:
-        group_values = _extract_ids_from_path_filter(path, "groups")
-    if op_type == "replace":
-        return set(group_values)
-    elif op_type == "add":
-        teams_set.update(group_values)
-    elif op_type == "remove":
-        for gid in group_values:
-            teams_set.discard(gid)
-    return None
+def _is_group_path(path: str | None) -> bool:
+    return isinstance(path, str) and path.lower().startswith("groups")
+
+
+def _apply_group_operation(teams: tuple[str, ...], op: SCIMPatchOperation) -> tuple[str, ...]:
+    listed: Final = _extract_group_values(op.value)
+    group_values: Final = listed if listed or op.value is not None else _extract_ids_from_path_filter(op.path, "groups")
+    if op.op == "replace":
+        return tuple(dict.fromkeys(group_values))
+    if op.op == "add":
+        return tuple(dict.fromkeys((*teams, *group_values)))
+    removed: Final = frozenset(group_values)
+    return tuple(team for team in teams if team not in removed)
+
+
+def _keep_record_team_order(record_teams: Sequence[str], target_teams: Sequence[str]) -> tuple[str, ...]:
+    target: Final = frozenset(target_teams)
+    record: Final = tuple(dict.fromkeys(record_teams))
+    known: Final = frozenset(record)
+    return (
+        *(team for team in record if team in target),
+        *(team for team in dict.fromkeys(target_teams) if team not in known),
+    )
 
 
 def _multi_valued_attribute_base(path: str) -> str:
@@ -2520,8 +2538,8 @@ def _handle_generic_metadata(path: str, op_type: str, value: object, metadata: d
 def _apply_patch_ops(
     existing_user: LiteLLM_UserTable,
     patch_ops: SCIMPatchOp,
-) -> tuple[dict[str, object], set[str]]:
-    """Apply patch operations and return update data and final team set."""
+) -> tuple[dict[str, object], tuple[str, ...]]:
+    """Apply patch operations and return update data and the final teams in order."""
     update_data: Final[dict[str, object]] = {}
     stored_metadata: Final[object] = TypeAdapter(object).validate_python(existing_user.model_dump().get("metadata"))
     metadata: Final = _JSON_OBJECT_ADAPTER.validate_python(stored_metadata) if isinstance(stored_metadata, dict) else {}
@@ -2541,9 +2559,6 @@ def _apply_patch_ops(
     if isinstance(enterprise, _InvalidEnterpriseValue):
         _raise_invalid_enterprise_value(enterprise)
 
-    teams_set: Final[set[str]] = set(existing_user.teams or [])
-    replace_team_set: set[str] | None = None
-
     for op in patch_ops.Operations:
         path = op.path.lower() if isinstance(op.path, str) else ""
         value = op.value
@@ -2585,16 +2600,20 @@ def _apply_patch_ops(
             _handle_name_update(path, op_type, value, scim_metadata)
         elif _multi_valued_attribute_base(path) in SCIM_MULTI_VALUED_ATTRIBUTE_METADATA_KEYS:
             _handle_multi_valued_attribute_update(path, op_type, value, metadata)
-        elif path.startswith("groups"):
-            new_replace_set = _handle_group_operations(op_type, value, teams_set, op.path)
-            if new_replace_set is not None:
-                replace_team_set = new_replace_set
-        elif _enterprise_extension_sub_path(path) is not None:
+        elif _is_group_path(path) or _enterprise_extension_sub_path(path) is not None:
             continue
         else:
             _handle_generic_metadata(path, op_type, value, metadata)
 
-    final_team_set: Final = replace_team_set if replace_team_set is not None else teams_set
+    record_teams: Final = tuple(existing_user.teams or [])
+    group_ops: Final = tuple(op for op in patch_ops.Operations if _is_group_path(op.path))
+    last_replace: Final = next((op for op in reversed(group_ops) if op.op == "replace"), None)
+    final_teams: Final = _keep_record_team_order(
+        record_teams,
+        _apply_group_operation(record_teams, last_replace)
+        if last_replace is not None
+        else reduce(_apply_group_operation, group_ops, record_teams),
+    )
     final_metadata: Final = (
         {
             **{key: metadata_value for key, metadata_value in metadata.items() if key != SCIM_ENTERPRISE_METADATA_KEY},
@@ -2605,7 +2624,7 @@ def _apply_patch_ops(
         else {**metadata, "scim_metadata": scim_metadata}
     )
     update_data["metadata"] = final_metadata
-    return update_data, final_team_set
+    return update_data, final_teams
 
 
 def _is_user_not_in_team_error(exc: HTTPException) -> bool:
@@ -2758,7 +2777,7 @@ async def patch_user(
 
         prev_active: Final = _user_scim_active(existing_user)
 
-        update_data, final_team_set = _apply_patch_ops(
+        update_data, final_teams = _apply_patch_ops(
             existing_user=existing_user,
             patch_ops=patch_ops,
         )
@@ -2770,15 +2789,15 @@ async def patch_user(
         await _handle_team_membership_changes(
             user_id=user_id,
             existing_teams=existing_user.teams or [],
-            new_teams=list(final_team_set),
+            new_teams=list(final_teams),
         )
 
-        update_data["teams"] = list(final_team_set)
+        update_data["teams"] = list(final_teams)
 
         admin_group: Final = await _get_scim_admin_group()
         if admin_group is not None:
             update_data["user_role"] = _resolve_scim_user_role(
-                await _scim_groups_from_team_ids(prisma_client, list(final_team_set)),
+                await _scim_groups_from_team_ids(prisma_client, list(final_teams)),
                 admin_group,
                 _default_scim_user_role(),
             )

litellm/proxy/management_endpoints/subscription_accounts.py

@@ -0,0 +1,335 @@
+"""
+Subscription accounts: the operator-entered recurring fee of a provider account whose
+requests are covered by a flat subscription, shown beside metered spend on the Usage page.
+"""
+
+import re
+from collections.abc import Callable, Iterable, Mapping
+from dataclasses import dataclass
+from datetime import date
+from itertools import chain
+from typing import TYPE_CHECKING, Annotated, Final, TypeAlias
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from pydantic import TypeAdapter
+
+from litellm.constants import LITELLM_PROXY_ADMIN_NAME, SUBSCRIPTION_BACKED_PROVIDERS
+from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
+from litellm.llms.subscription_accounts import resolve_subscription_account_id
+from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
+from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth, user_api_key_has_admin_view
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.spend_tracking.subscription_billing_periods import billing_periods_starting_within
+from litellm.proxy.utils import get_prisma_client_or_throw
+from litellm.repositories.subscription_account_repository import (
+    DuplicateSubscriptionAccount,
+    KeepLabel,
+    SetLabel,
+    SubscriptionAccountFeeChanges,
+    SubscriptionAccountFeeFields,
+    SubscriptionAccountRepository,
+    SubscriptionAccountStore,
+)
+from litellm.types.proxy.management_endpoints.subscription_accounts import (
+    BillingPeriod,
+    DeleteSubscriptionAccountRequest,
+    SubscriptionAccountFeeRequest,
+    SubscriptionAccountUsage,
+    SubscriptionUsageResponse,
+    UpdateSubscriptionAccountFeeRequest,
+)
+from litellm.types.router import DeploymentTypedDict
+
+if TYPE_CHECKING:
+    from litellm.router import Router
+
+router: Final = APIRouter(prefix="/subscription_accounts", tags=["subscription accounts"])
+Auth: TypeAlias = Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)]
+
+
+@dataclass(frozen=True, slots=True)
+class SubscriptionDeployment:
+    model_name: str
+    custom_llm_provider: str
+    account_id: str | None
+
+
+AccountIdResolver: TypeAlias = Callable[[str, Mapping[str, object]], str | None]
+_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+
+
+def _declared_provider(litellm_params: Mapping[str, object]) -> str | None:
+    model: Final = litellm_params.get("model")
+    configured_provider: Final = litellm_params.get("custom_llm_provider")
+    return declared_authenticating_provider(
+        model if isinstance(model, str) else None,
+        configured_provider if isinstance(configured_provider, str) else None,
+    )
+
+
+def _model_info(deployment: Mapping[str, object]) -> Mapping[str, object]:
+    return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
+
+
+def _public_model_name(entry: DeploymentTypedDict) -> str:
+    team_public_model_name: Final = _model_info(entry).get("team_public_model_name")
+    return team_public_model_name if isinstance(team_public_model_name, str) else entry["model_name"]
+
+
+def _subscription_deployment(
+    entry: DeploymentTypedDict, account_id_resolver: AccountIdResolver
+) -> SubscriptionDeployment | None:
+    litellm_params: Final = entry["litellm_params"]
+    provider: Final = _declared_provider(litellm_params)
+    if provider is None or provider not in SUBSCRIPTION_BACKED_PROVIDERS:
+        return None
+    return SubscriptionDeployment(
+        model_name=_public_model_name(entry),
+        custom_llm_provider=provider,
+        account_id=account_id_resolver(provider, litellm_params),
+    )
+
+
+def subscription_deployments_from_model_list(
+    model_list: Iterable[DeploymentTypedDict], account_id_resolver: AccountIdResolver
+) -> tuple[SubscriptionDeployment, ...]:
+    candidates: Final = (_subscription_deployment(entry, account_id_resolver) for entry in model_list)
+    return tuple(candidate for candidate in candidates if candidate is not None)
+
+
+def subscription_deployments_of_router(
+    llm_router: "Router | None", account_id_resolver: AccountIdResolver
+) -> tuple[SubscriptionDeployment, ...]:
+    if llm_router is None:
+        return ()
+    registry: Final = llm_router.registry()
+    indexed_names: Final = tuple(registry.index("model_name"))
+    deployments: Final = chain.from_iterable(registry.deployments(name, source="named") or () for name in indexed_names)
+    return subscription_deployments_from_model_list(deployments, account_id_resolver)
+
+
+def get_subscription_deployments() -> tuple[SubscriptionDeployment, ...]:
+    from litellm.proxy.proxy_server import llm_router
+
+    return subscription_deployments_of_router(llm_router, resolve_subscription_account_id)
+
+
+def get_subscription_account_repository() -> SubscriptionAccountStore:
+    return SubscriptionAccountRepository(get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value))
+
+
+Store: TypeAlias = Annotated[SubscriptionAccountStore, Depends(get_subscription_account_repository)]
+Deployments: TypeAlias = Annotated[tuple[SubscriptionDeployment, ...], Depends(get_subscription_deployments)]
+
+
+def _forbidden(user_api_key_dict: UserAPIKeyAuth) -> HTTPException:
+    return HTTPException(
+        status_code=status.HTTP_403_FORBIDDEN,
+        detail={"error": f"{CommonProxyErrors.not_allowed_access.value}, your role={user_api_key_dict.user_role}"},
+    )
+
+
+def _account_usage(
+    custom_llm_provider: str,
+    account_id: str | None,
+    deployments: Iterable[SubscriptionDeployment],
+    fee_row: LiteLLM_SubscriptionAccountTable | None,
+    start_date: date,
+    end_date: date,
+) -> SubscriptionAccountUsage:
+    model_names: Final = tuple(sorted({deployment.model_name for deployment in deployments}))
+    if fee_row is None:
+        return SubscriptionAccountUsage(
+            custom_llm_provider=custom_llm_provider,
+            account_id=account_id,
+            deployments=model_names,
+            fee=None,
+            billing_periods=(),
+            fixed_cost=None,
+        )
+    periods: Final = billing_periods_starting_within(
+        date.fromisoformat(fee_row.billing_period_start), start_date, end_date
+    )
+    return SubscriptionAccountUsage(
+        custom_llm_provider=custom_llm_provider,
+        account_id=account_id,
+        deployments=model_names,
+        fee=fee_row,
+        billing_periods=tuple(BillingPeriod(start=period.start, end=period.end) for period in periods),
+        fixed_cost=len(periods) * fee_row.monthly_fee,
+    )
+
+
+def build_subscription_usage(
+    deployments: Iterable[SubscriptionDeployment],
+    fee_rows: Iterable[LiteLLM_SubscriptionAccountTable],
+    start_date: date,
+    end_date: date,
+) -> SubscriptionUsageResponse:
+    deployment_tuple: Final = tuple(deployments)
+    fees_by_account: Final = {(row.custom_llm_provider, row.account_id): row for row in fee_rows}
+    deployment_keys: Final = {(d.custom_llm_provider, d.account_id) for d in deployment_tuple}
+    account_keys: Final = sorted(deployment_keys | set(fees_by_account), key=lambda key: (key[0], key[1] or "~"))
+    accounts: Final = tuple(
+        _account_usage(
+            provider,
+            account_id,
+            (d for d in deployment_tuple if (d.custom_llm_provider, d.account_id) == (provider, account_id)),
+            fees_by_account.get((provider, account_id)) if account_id is not None else None,
+            start_date,
+            end_date,
+        )
+        for provider, account_id in account_keys
+    )
+    return SubscriptionUsageResponse(
+        start_date=start_date,
+        end_date=end_date,
+        subscription_providers=tuple(sorted(SUBSCRIPTION_BACKED_PROVIDERS)),
+        accounts=accounts,
+    )
+
+
+_DAY_PATTERN: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
+
+
+def _parse_day(value: str) -> date | None:
+    if _DAY_PATTERN.fullmatch(value) is None:
+        return None
+    try:
+        return date.fromisoformat(value)
+    except ValueError:
+        return None
+
+
+def _parse_date(value: str, name: str) -> date:
+    parsed: Final = _parse_day(value)
+    if parsed is None:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail={"error": f"{name} must be YYYY-MM-DD, got {value!r}"}
+        )
+    return parsed
+
+
+@router.get("/usage", response_model=SubscriptionUsageResponse, dependencies=[Depends(user_api_key_auth)])
+async def get_subscription_usage(
+    start_date: Annotated[str, Query(description="First day of the Usage page filter, YYYY-MM-DD")],
+    end_date: Annotated[str, Query(description="Last day of the Usage page filter, YYYY-MM-DD")],
+    user_api_key_dict: Auth,
+    repository: Store,
+    deployments: Deployments,
+) -> SubscriptionUsageResponse:
+    """
+    Subscription-backed accounts behind the proxy's deployments or carrying a stored fee, each with
+    its configured monthly fee and the fixed cost attributed to the date range: the fee once per billing
+    period whose start date falls inside the range, however many deployments share the account.
+    """
+    if not user_api_key_has_admin_view(user_api_key_dict):
+        raise _forbidden(user_api_key_dict)
+    start: Final = _parse_date(start_date, "start_date")
+    end: Final = _parse_date(end_date, "end_date")
+    if end < start:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail={"error": "end_date must not be before start_date"}
+        )
+    return build_subscription_usage(deployments, await repository.list_all(), start, end)
+
+
+def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None:
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise _forbidden(user_api_key_dict)
+
+
+def _actor(user_api_key_dict: UserAPIKeyAuth) -> str:
+    return user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME
+
+
+@router.post("/new", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)])
+async def new_subscription_account(
+    request: SubscriptionAccountFeeRequest,
+    user_api_key_dict: Auth,
+    repository: Store,
+) -> LiteLLM_SubscriptionAccountTable:
+    """Record the monthly fee of one subscription-backed provider account."""
+    _require_proxy_admin(user_api_key_dict)
+    if request.custom_llm_provider not in SUBSCRIPTION_BACKED_PROVIDERS:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={
+                "error": f"{request.custom_llm_provider!r} is not a subscription-backed provider; "
+                f"expected one of {sorted(SUBSCRIPTION_BACKED_PROVIDERS)}"
+            },
+        )
+    created: Final = await repository.create_account(
+        request.custom_llm_provider,
+        request.account_id,
+        SubscriptionAccountFeeFields(
+            monthly_fee=request.monthly_fee,
+            currency=request.currency,
+            billing_period_start=request.billing_period_start.isoformat(),
+            label=request.label,
+        ),
+        _actor(user_api_key_dict),
+    )
+    if isinstance(created, DuplicateSubscriptionAccount):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={
+                "error": f"A fee is already configured for {created.custom_llm_provider} account "
+                f"{created.account_id}; use /subscription_accounts/update to change it"
+            },
+        )
+    return created
+
+
+def _fee_changes(request: UpdateSubscriptionAccountFeeRequest) -> SubscriptionAccountFeeChanges:
+    return SubscriptionAccountFeeChanges(
+        monthly_fee=request.monthly_fee,
+        currency=request.currency,
+        billing_period_start=(
+            request.billing_period_start.isoformat() if request.billing_period_start is not None else None
+        ),
+        label=SetLabel(request.label) if "label" in request.model_fields_set else KeepLabel(),
+    )
+
+
+@router.post("/update", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)])
+async def update_subscription_account(
+    request: UpdateSubscriptionAccountFeeRequest,
+    user_api_key_dict: Auth,
+    repository: Store,
+) -> LiteLLM_SubscriptionAccountTable:
+    """
+    Change the fee, currency, billing period start, or label of a subscription account.
+    Only the fields named in the request are written, so two admins changing different
+    fields at the same time both land.
+    """
+    _require_proxy_admin(user_api_key_dict)
+    updated: Final = await repository.update_fee(
+        request.subscription_account_id, _fee_changes(request), _actor(user_api_key_dict)
+    )
+    if updated is None:
+        raise _not_found(request.subscription_account_id)
+    return updated
+
+
+@router.post("/delete", response_model=LiteLLM_SubscriptionAccountTable, dependencies=[Depends(user_api_key_auth)])
+async def delete_subscription_account(
+    request: DeleteSubscriptionAccountRequest,
+    user_api_key_dict: Auth,
+    repository: Store,
+) -> LiteLLM_SubscriptionAccountTable:
+    """Stop tracking a subscription account's fee; its requests stay marked as subscription-covered."""
+    _require_proxy_admin(user_api_key_dict)
+    if await repository.find_by_id(request.subscription_account_id) is None:
+        raise _not_found(request.subscription_account_id)
+    deleted: Final = await repository.delete_account(request.subscription_account_id)
+    if deleted is None:
+        raise _not_found(request.subscription_account_id)
+    return deleted
+
+
+def _not_found(subscription_account_id: str) -> HTTPException:
+    return HTTPException(
+        status_code=status.HTTP_404_NOT_FOUND,
+        detail={"error": f"No subscription account with id {subscription_account_id}"},
+    )

litellm/proxy/management_endpoints/ui_sso.py

@@ -8,7 +8,6 @@ Has all /sso/* routes
 /sso/debug/callback - returns the OpenID object returned by the SSO provider
 """
 
-import asyncio
 import base64
 import hashlib
 import inspect
@@ -151,6 +150,7 @@ from litellm.proxy.utils import (
     get_custom_url,
     get_server_root_path,
 )
+from litellm.repositories.chunked_in import find_many_in
 from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import SSOConfigRepository
 from litellm.repositories.team_repository import TeamRepository
@@ -1855,7 +1855,7 @@ async def warn_if_id_jag_capture_gap(*, retention_enabled: RetentionCheck | None
     verbose_proxy_logger.warning("SSO debug callback ran with an oauth2_id_jag capture gap: %s", gap)
 
 
-async def create_team_member_add_task(team_id, user_info):
+async def create_team_member_add_task(team_id: str, user_info: NewUserResponse | LiteLLM_UserTable):
     """Create a task for adding a member to a team."""
     try:
         member: Final = Member(user_id=user_info.user_id, role="user")
@@ -1871,22 +1871,43 @@ async def create_team_member_add_task(team_id, user_info):
         verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e)
 
 
-async def add_missing_team_member(user_info: NewUserResponse | LiteLLM_UserTable, sso_teams: list[str]):
+async def find_existing_team_ids(team_ids: Sequence[str]) -> frozenset[str] | None:
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        return None
+    try:
+        rows: Final = await find_many_in(TeamRepository(prisma_client, use_writer=True).table, "team_id", team_ids)
+    except Exception as e:
+        verbose_proxy_logger.debug("[Non-Blocking] Error looking up SSO teams: %s", e)
+        return None
+    return frozenset(row.team_id for row in rows)
+
+
+async def add_missing_team_member(
+    user_info: NewUserResponse | LiteLLM_UserTable,
+    sso_teams: Sequence[str],
+    *,
+    add_team_member: Callable[
+        [str, NewUserResponse | LiteLLM_UserTable], Awaitable[object]
+    ] = create_team_member_add_task,
+    existing_team_ids: Callable[[Sequence[str]], Awaitable[frozenset[str] | None]] = find_existing_team_ids,
+) -> None:
     """
     - Get missing teams (diff b/w user_info.team_ids and sso_teams)
-    - Add missing user to missing teams
+    - Drop the IdP groups that are not teams with one lookup, since group claims often carry many of them
+    - Add missing user to missing teams, one at a time in sso_teams order, so user.teams keeps the claim's order
     """
-    # Handle None as empty list for new users
-    user_teams: Final = user_info.teams if user_info.teams is not None else []
-    missing_teams: Final = set(sso_teams) - set(user_teams)
-    missing_teams_list: Final = list(missing_teams)
-    tasks = []
-    tasks = [create_team_member_add_task(team_id, user_info) for team_id in missing_teams_list]
-
-    try:
-        await asyncio.gather(*tasks)
-    except Exception as e:
-        verbose_proxy_logger.debug("[Non-Blocking] Error trying to add sso user to db: %s", e)
+    held_teams: Final = frozenset(user_info.teams or ())
+    missing_teams: Final = tuple(team_id for team_id in dict.fromkeys(sso_teams) if team_id not in held_teams)
+    if not missing_teams:
+        return
+    existing: Final = await existing_team_ids(missing_teams)
+    teams_to_add: Final = (
+        missing_teams if existing is None else tuple(team_id for team_id in missing_teams if team_id in existing)
+    )
+    for team_id in teams_to_add:
+        await add_team_member(team_id, user_info)
 
 
 def get_disabled_non_admin_personal_key_creation():
@@ -2451,7 +2472,9 @@ async def fetch_cli_sso_team_details(
     except Exception as e:
         verbose_proxy_logger.error("Error fetching team details for CLI SSO session: %s", e)
         return None
-    return tuple(_cli_sso_team_detail(team_row.model_dump()) for team_row in prisma_teams)
+    fetched_details: Final = tuple(_cli_sso_team_detail(team_row.model_dump()) for team_row in prisma_teams)
+    details_by_team_id: Final = MappingProxyType({detail.team_id: detail for detail in fetched_details})
+    return tuple(details_by_team_id[team_id] for team_id in dict.fromkeys(teams) if team_id in details_by_team_id)
 
 
 def _cli_sso_session_teams(team_details: Sequence[CliSsoTeamDetail]) -> list[str]:

litellm/proxy/proxy_server.py

@@ -549,6 +549,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import (
     write_reload_interval,
 )
 from litellm.proxy.common_utils.proxy_state import ProxyState
+from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
 from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob
 from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState
 from litellm.proxy.common_utils.scheduled_job_stagger import (
@@ -717,6 +718,9 @@ from litellm.proxy.management_endpoints.customer_endpoints import (
     router as customer_router,
 )
 from litellm.proxy.management_endpoints.daily_activity_routes import router as daily_activity_router
+from litellm.proxy.management_endpoints.fallback_management_endpoints import (
+    apply_content_policy_fallbacks,
+)
 from litellm.proxy.management_endpoints.fallback_management_endpoints import (
     router as fallback_management_router,
 )
@@ -776,6 +780,9 @@ from litellm.proxy.management_endpoints.session_endpoints import (
     router as session_management_router,
 )
 from litellm.proxy.management_endpoints.sso.saml_sso import SAML_AUTHN_REQUEST_TTL_SECONDS
+from litellm.proxy.management_endpoints.subscription_accounts import (
+    router as subscription_accounts_router,
+)
 from litellm.proxy.management_endpoints.tag_management_endpoints import (
     router as tag_management_router,
 )
@@ -8159,14 +8166,38 @@ class ProxyConfig:
             if db_router_settings is not None and db_router_settings.param_value is not None
             else _EMPTY_SETTINGS_MAPPING
         )
+        previous_content_policy_rules: Final = self.router_settings.resolved().get("content_policy_fallbacks")
         self.router_settings.apply_db_row("router_settings", db_values)
         combined_router_settings: Final = self.router_settings.resolved()
         if combined_router_settings:
-            self._apply_router_settings(llm_router, combined_router_settings)
+            self._apply_router_settings(
+                llm_router,
+                combined_router_settings,
+                content_policy_owned_by_config=self.router_settings.owned_by_config("content_policy_fallbacks"),
+                content_policy_rules_changed=(
+                    combined_router_settings.get("content_policy_fallbacks") != previous_content_policy_rules
+                ),
+            )
 
     @staticmethod
-    def _apply_router_settings(llm_router: Router, router_settings: Mapping[str, object]) -> None:
-        llm_router.apply(ChangeSet(settings={k: v for k, v in router_settings.items() if k != "routing_groups"}))
+    def _apply_router_settings(
+        llm_router: Router,
+        router_settings: Mapping[str, object],
+        *,
+        content_policy_owned_by_config: bool,
+        content_policy_rules_changed: bool,
+    ) -> None:
+        llm_router.apply(
+            ChangeSet(
+                settings={
+                    k: v for k, v in router_settings.items() if k not in ("routing_groups", "content_policy_fallbacks")
+                }
+            )
+        )
+        if "content_policy_fallbacks" in router_settings and not content_policy_owned_by_config:
+            apply_content_policy_fallbacks(
+                llm_router, router_settings["content_policy_fallbacks"], log_skipped_rules=content_policy_rules_changed
+            )
         if "routing_groups" not in router_settings:
             return
         try:
@@ -8750,6 +8781,7 @@ class ProxyConfig:
         subscriber: Final = ConfigSyncSubscriber(
             redis_cache=redis_cache,
             resync_callbacks=(_resync_config_from_db,),
+            apply_model_ids=resync_model_deployments_by_ids,
         )
         self.config_sync_subscriber = subscriber
         subscriber.start()
@@ -21767,6 +21799,7 @@ app.include_router(memory_router)
 app.include_router(lens_router)
 app.include_router(plugin_router)
 app.include_router(cost_tracking_settings_router)
+app.include_router(subscription_accounts_router)
 app.include_router(prompt_caching_requests_router)
 app.include_router(router_settings_router)
 app.include_router(fallback_management_router)

litellm/proxy/schema.prisma

@@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
   @@id([event_id, target, version])
   @@index([target, state, next_attempt_at])
 }
+
+model LiteLLM_SubscriptionAccountTable {
+  subscription_account_id String @id @default(uuid())
+  custom_llm_provider String
+  account_id String
+  label String?
+  monthly_fee Float
+  currency String
+  billing_period_start String
+  created_at    DateTime               @default(now()) @map("created_at")
+  created_by String
+  updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")
+  updated_by String
+
+  @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
+}

litellm/proxy/spend_tracking/subscription_billing_periods.py

@@ -0,0 +1,54 @@
+from calendar import monthrange
+from dataclasses import dataclass
+from datetime import date, timedelta
+from typing import Final
+
+
+@dataclass(frozen=True, slots=True)
+class BillingPeriodRange:
+    start: date
+    end: date
+
+
+def _month_index(anchor: date, months_after_anchor: int) -> int:
+    return anchor.year * 12 + (anchor.month - 1) + months_after_anchor
+
+
+_LAST_MONTH_INDEX: Final = _month_index(date.max, 0)
+
+
+def _month_start(anchor: date, months_after_anchor: int) -> date:
+    month_index: Final = _month_index(anchor, months_after_anchor)
+    year: Final = month_index // 12
+    month: Final = month_index % 12 + 1
+    return date(year, month, min(anchor.day, monthrange(year, month)[1]))
+
+
+def _period_end(anchor: date, months_after_anchor: int) -> date:
+    if _month_index(anchor, months_after_anchor + 1) > _LAST_MONTH_INDEX:
+        return date.max
+    return _month_start(anchor, months_after_anchor + 1) - timedelta(days=1)
+
+
+def _first_candidate_index(anchor: date, start_date: date) -> int:
+    months_apart: Final = (start_date.year - anchor.year) * 12 + (start_date.month - anchor.month)
+    return max(0, months_apart - 1)
+
+
+def billing_periods_starting_within(anchor: date, start_date: date, end_date: date) -> tuple[BillingPeriodRange, ...]:
+    """Monthly periods anchored on `anchor`'s day of month whose start lies inside [start_date, end_date]."""
+    if end_date < start_date or end_date < anchor:
+        return ()
+    first_index: Final = _first_candidate_index(anchor, start_date)
+    last_index: Final = min(
+        first_index + _month_span(start_date, end_date) + 1, _LAST_MONTH_INDEX - _month_index(anchor, 0)
+    )
+    candidates: Final = (
+        BillingPeriodRange(start=_month_start(anchor, index), end=_period_end(anchor, index))
+        for index in range(first_index, last_index + 1)
+    )
+    return tuple(period for period in candidates if start_date <= period.start <= end_date)
+
+
+def _month_span(start_date: date, end_date: date) -> int:
+    return (end_date.year - start_date.year) * 12 + (end_date.month - start_date.month)

litellm/repositories/subscription_account_repository.py

@@ -0,0 +1,134 @@
+"""
+Subscription account repository for database operations on LiteLLM_SubscriptionAccountTable.
+"""
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
+
+from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
+from litellm.repositories.base_repository import BaseRepository, is_unique_violation
+from litellm.repositories.prisma_protocols import TableActions
+from litellm.repositories.table_repositories import PrismaTableRepository
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
+
+@dataclass(frozen=True, slots=True)
+class DuplicateSubscriptionAccount:
+    custom_llm_provider: str
+    account_id: str
+
+
+@dataclass(frozen=True, slots=True)
+class SubscriptionAccountFeeFields:
+    monthly_fee: float
+    currency: str
+    billing_period_start: str
+    label: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class KeepLabel:
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class SetLabel:
+    value: str | None
+
+
+LabelChange: TypeAlias = KeepLabel | SetLabel
+
+
+@dataclass(frozen=True, slots=True)
+class SubscriptionAccountFeeChanges:
+    monthly_fee: float | None = None
+    currency: str | None = None
+    billing_period_start: str | None = None
+    label: LabelChange = KeepLabel()
+
+    def columns(self) -> Mapping[str, float | str | None]:
+        named: Final = {
+            "monthly_fee": self.monthly_fee,
+            "currency": self.currency,
+            "billing_period_start": self.billing_period_start,
+        }
+        present: Final = {column: value for column, value in named.items() if value is not None}
+        label_column: Final = {"label": self.label.value} if isinstance(self.label, SetLabel) else {}
+        return MappingProxyType({**present, **label_column})
+
+
+class SubscriptionAccountStore(Protocol):
+    async def list_all(self) -> tuple[LiteLLM_SubscriptionAccountTable, ...]: ...
+
+    async def find_by_id(self, id_value: str) -> LiteLLM_SubscriptionAccountTable | None: ...
+
+    async def create_account(
+        self, custom_llm_provider: str, account_id: str, fee: SubscriptionAccountFeeFields, created_by: str
+    ) -> LiteLLM_SubscriptionAccountTable | DuplicateSubscriptionAccount: ...
+
+    async def update_fee(
+        self, subscription_account_id: str, changes: SubscriptionAccountFeeChanges, updated_by: str
+    ) -> LiteLLM_SubscriptionAccountTable | None: ...
+
+    async def delete_account(self, subscription_account_id: str) -> LiteLLM_SubscriptionAccountTable | None: ...
+
+
+class _SubscriptionAccountTable(PrismaTableRepository["prisma_models.LiteLLM_SubscriptionAccountTable"]):
+    table_name = "litellm_subscriptionaccounttable"
+
+
+class SubscriptionAccountRepository(BaseRepository[LiteLLM_SubscriptionAccountTable]):
+    def __init__(self, prisma_client: object) -> None:
+        super().__init__(prisma_client)
+        self._table: Final = _SubscriptionAccountTable(prisma_client)
+
+    @property
+    def table(self) -> TableActions["prisma_models.LiteLLM_SubscriptionAccountTable"]:
+        return self._table.table
+
+    @property
+    def model_class(self) -> type[LiteLLM_SubscriptionAccountTable]:
+        return LiteLLM_SubscriptionAccountTable
+
+    async def list_all(self) -> tuple[LiteLLM_SubscriptionAccountTable, ...]:
+        return tuple(await self.find_many(order={"created_at": "asc"}))
+
+    async def find_by_id(
+        self, id_value: str, id_field: str = "subscription_account_id"
+    ) -> LiteLLM_SubscriptionAccountTable | None:
+        return await super().find_by_id(id_value, id_field)
+
+    async def create_account(
+        self, custom_llm_provider: str, account_id: str, fee: SubscriptionAccountFeeFields, created_by: str
+    ) -> LiteLLM_SubscriptionAccountTable | DuplicateSubscriptionAccount:
+        try:
+            return await self.create(
+                {
+                    "custom_llm_provider": custom_llm_provider,
+                    "account_id": account_id,
+                    "monthly_fee": fee.monthly_fee,
+                    "currency": fee.currency,
+                    "billing_period_start": fee.billing_period_start,
+                    "label": fee.label,
+                    "created_by": created_by,
+                    "updated_by": created_by,
+                }
+            )
+        except Exception as exc:
+            if is_unique_violation(exc):
+                return DuplicateSubscriptionAccount(custom_llm_provider=custom_llm_provider, account_id=account_id)
+            raise
+
+    async def update_fee(
+        self, subscription_account_id: str, changes: SubscriptionAccountFeeChanges, updated_by: str
+    ) -> LiteLLM_SubscriptionAccountTable | None:
+        return await self.update(
+            subscription_account_id, {**changes.columns(), "updated_by": updated_by}, id_field="subscription_account_id"
+        )
+
+    async def delete_account(self, subscription_account_id: str) -> LiteLLM_SubscriptionAccountTable | None:
+        return await self.delete(subscription_account_id, id_field="subscription_account_id")

litellm/responses/main.py

@@ -31,7 +31,10 @@ from litellm.integrations.anthropic_cache_control_hook import (
 from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
-from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
+from litellm.litellm_core_utils.get_llm_provider_logic import (
+    declared_authenticating_provider,
+    get_provider_affinity_header,
+)
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
@@ -805,6 +808,7 @@ def provider_stores_the_conversation(model: str, custom_llm_provider: str | None
     return not _will_bridge_to_chat_completions(
         model,
         custom_llm_provider
+        or declared_authenticating_provider(model)
         or litellm.get_llm_provider(model=model, api_base=api_base, litellm_params=litellm_params)[1],
         bool(kwargs.get("use_chat_completions_api")),
         kwargs.get("model_info"),

litellm/responses/utils.py

@@ -1,10 +1,11 @@
 import base64
 import os
 import re
-from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
+from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from functools import reduce
 from types import MappingProxyType
-from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload
+from typing import Any, Final, NoReturn, Optional, TypeVar, Union, cast, get_type_hints, overload
 
 from pydantic import BaseModel, TypeAdapter
 from typing_extensions import TypedDict as ExtraItemsTypedDict
@@ -15,7 +16,13 @@ from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items
 from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path
 from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
-from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
+from litellm.llms.base_llm.responses.transformation import (
+    ANY_REASONING_FIELD_VALUES,
+    REASONING_SUMMARY_VALUES,
+    BaseResponsesAPIConfig,
+    ReasoningFieldValues,
+    ReasoningSummaryLevel,
+)
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
 from litellm.types.llms.openai import (
     AllMessageValues,
@@ -1411,9 +1418,6 @@ def reasoning_with_summary_default(
     return {**reasoning} if summary is None else {**reasoning, "summary": summary}
 
 
-_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
-
-
 def _openai_model_info(model: str) -> ModelInfo | None:
     from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
 
@@ -1425,24 +1429,115 @@ def _declared_values(values: Sequence[str] | None) -> frozenset[str | None] | No
     return None if values is None else frozenset(values)
 
 
-def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | None] | None]:
+_REFUSABLE_REASONING_FIELDS: Final = frozenset({"mode", "context"})
+
+
+def _openai_reasoning_field_values(model: str) -> ReasoningFieldValues:
     model_info: Final = _openai_model_info(model)
     if model_info is None:
-        return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None})
+        return ANY_REASONING_FIELD_VALUES
     if not model_info.get("supports_reasoning"):
         return MappingProxyType({})
     return MappingProxyType(
         {
-            "summary": _SUMMARY_VALUES,
+            "summary": REASONING_SUMMARY_VALUES,
             "mode": _declared_values(model_info.get("reasoning_modes")),
             "context": _declared_values(model_info.get("reasoning_contexts")),
         }
     )
 
 
-def _openai_accepts_reasoning_field(
-    key: str, value: object, values_by_field: Mapping[str, frozenset[str | None] | None]
-) -> bool:
+_CATALOG_REASONING_FIELDS: Final = (("mode", "reasoning_modes"), ("context", "reasoning_contexts"))
+
+
+def _catalog_values(values: object) -> frozenset[str | None]:
+    return frozenset(value for value in iter_items(values) if isinstance(value, str))
+
+
+def _declared_reasoning_values(row: Mapping[str, object] | None) -> Mapping[str, frozenset[str | None]]:
+    if row is None:
+        return {}
+    return {
+        field: _catalog_values(row[key]) for field, key in _CATALOG_REASONING_FIELDS if isinstance(row.get(key), list)
+    }
+
+
+def _with_declared_reasoning_values(
+    defaults: ReasoningFieldValues | None,
+    model: str,
+    custom_llm_provider: str,
+    deployment_model_info: Mapping[str, object] | None,
+) -> ReasoningFieldValues | None:
+    catalog: Final = cast(Mapping[str, object], litellm.model_cost)  # cast-ok: read-only view of the legacy registry
+    catalog_row: Final = as_str_mapping(catalog.get(f"{custom_llm_provider}/{model.removeprefix('responses/')}"))
+    overrides: Final = {
+        **_declared_reasoning_values(catalog_row),
+        **_declared_reasoning_values(deployment_model_info),
+    }
+    if not overrides:
+        return defaults
+    return MappingProxyType({**(ANY_REASONING_FIELD_VALUES if defaults is None else defaults), **overrides})
+
+
+def _is_version_number(part: str) -> bool:
+    return part.isdigit() and len(part) <= 2
+
+
+def _splits_a_version(parts: Sequence[str], end: int) -> bool:
+    return end < len(parts) and _is_version_number(parts[end - 1]) and _is_version_number(parts[end])
+
+
+def _hyphen_runs_longest_first(name: str) -> Iterator[str]:
+    parts: Final = name.rsplit("/", 1)[-1].split("-")
+    for length in range(len(parts), 0, -1):
+        yield from (
+            "-".join(parts[start : start + length])
+            for start in range(len(parts) - length + 1)
+            if not _splits_a_version(parts, start + length)
+        )
+
+
+def _openai_catalog_family(name: str) -> str | None:
+    runs: Final = _hyphen_runs_longest_first(name)
+    return next((run for run in runs if OpenAIGPTConfig.is_openai_catalog_model(run)), None)
+
+
+def _openai_catalog_name(name: str) -> str | None:
+    last_segment: Final = name.rsplit("/", 1)[-1]
+    return last_segment if OpenAIGPTConfig.is_openai_catalog_model(last_segment) else None
+
+
+def _openai_catalog_model(model: str, base_model: str | None, names_wrap_models: bool) -> str | None:
+    find: Final = _openai_catalog_family if names_wrap_models else _openai_catalog_name
+    families: Final = (find(name) for name in (base_model, model) if name)
+    return next((family for family in families if family is not None), None)
+
+
+def _default_reasoning_field_values(
+    model: str, base_model: str | None, responses_config: BaseResponsesAPIConfig | None
+) -> ReasoningFieldValues | None:
+    declared: Final = None if responses_config is None else responses_config.reasoning_field_values()
+    if declared is not None:
+        return declared
+    names_wrap_models: Final = responses_config is not None and responses_config.deployment_names_wrap_openai_models()
+    catalog_model: Final = _openai_catalog_model(model, base_model, names_wrap_models)
+    if catalog_model is not None:
+        return _openai_reasoning_field_values(catalog_model)
+    return None if responses_config is None else responses_config.unresolved_model_reasoning_field_values(model)
+
+
+def _reasoning_field_values(
+    model: str,
+    base_model: str | None,
+    custom_llm_provider: str,
+    responses_config: BaseResponsesAPIConfig | None,
+    deployment_model_info: Mapping[str, object] | None,
+) -> ReasoningFieldValues | None:
+    defaults: Final = _default_reasoning_field_values(model, base_model, responses_config)
+    return _with_declared_reasoning_values(defaults, model, custom_llm_provider, deployment_model_info)
+
+
+def _accepts_reasoning_field(key: str, value: object, values_by_field: ReasoningFieldValues) -> bool:
     if key == "effort":
         return True
     if key not in values_by_field:
@@ -1451,11 +1546,8 @@ def _openai_accepts_reasoning_field(
     return allowed is None or (isinstance(value, str | None) and value in allowed)
 
 
-def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]:
-    values_by_field: Final = _openai_reasoning_field_values(model)
-    return {
-        key: value for key, value in reasoning.items() if _openai_accepts_reasoning_field(key, value, values_by_field)
-    }
+def _accepted_reasoning(reasoning: Mapping[str, object], values_by_field: ReasoningFieldValues) -> Mapping[str, object]:
+    return {key: value for key, value in reasoning.items() if _accepts_reasoning_field(key, value, values_by_field)}
 
 
 def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]:
@@ -1464,26 +1556,66 @@ def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]:
     return tuple(path for path in iter_items(additional_drop_params) if isinstance(path, str))
 
 
-def openai_bridge_reasoning(
+@dataclass(frozen=True, slots=True)
+class RefusedReasoningFields:
+    refused: tuple[str, ...]
+
+
+def raise_refused_reasoning_fields(refusal: RefusedReasoningFields, model: str, custom_llm_provider: str) -> NoReturn:
+    raise litellm.UnsupportedParamsError(
+        message=(
+            f"{custom_llm_provider} does not support parameters: "
+            f"{[f'reasoning_effort.{key}' for key in refusal.refused]}, for model={model}. "
+            "To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n"
+        ),
+        llm_provider=custom_llm_provider,
+        model=model,
+    )
+
+
+def bridge_reasoning(
     *,
     requested: Mapping[str, object],
     effort_kept: bool,
     summary_alias: object,
     model: str,
+    base_model: str | None,
+    custom_llm_provider: str,
+    api_base: str | None,
+    deployment_model_info: Mapping[str, object] | None,
     drop_params: bool,
     additional_drop_params: object,
-) -> Mapping[str, object] | None:
+) -> Mapping[str, object] | RefusedReasoningFields | None:
+    from litellm.utils import ProviderConfigManager
+
     drop_paths: Final = _additional_drop_paths(additional_drop_params)
     if "reasoning_effort" in drop_paths:
         return None
     kept: Final = requested if effort_kept else {key: value for key, value in requested.items() if key != "effort"}
-    with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, "openai")
-    accepted: Final = (
-        _openai_accepted_reasoning(with_summary, model)
-        if drop_params and OpenAIGPTConfig.is_openai_catalog_model(model)
-        else {**with_summary}
+    with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, custom_llm_provider)
+    sent: Final = (
+        as_str_mapping(
+            _apply_nested_drop_params({"reasoning_effort": {**with_summary}}, list(drop_paths)).get("reasoning_effort")
+        )
+        or {}
     )
-    remaining: Final = as_str_mapping(
-        _apply_nested_drop_params({"reasoning_effort": accepted}, list(drop_paths)).get("reasoning_effort")
+    responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(
+        custom_llm_provider, model, api_base
     )
-    return remaining or None
+    values_by_field: Final = _reasoning_field_values(
+        model, base_model, custom_llm_provider, responses_config, deployment_model_info
+    )
+    accepted: Final = sent if values_by_field is None else _accepted_reasoning(sent, values_by_field)
+    refused: Final = tuple(
+        key
+        for key, value in sent.items()
+        if key in _REFUSABLE_REASONING_FIELDS and key not in accepted and value is not None
+    )
+    if (
+        refused
+        and not drop_params
+        and responses_config is not None
+        and responses_config.ignores_unsupported_reasoning_fields()
+    ):
+        return RefusedReasoningFields(refused)
+    return (accepted if drop_params else sent) or None

litellm/router.py

@@ -255,6 +255,7 @@ from litellm.router_utils.clientside_credential_handler import (
 from litellm.router_utils.common_utils import (
     MID_STREAM_CONTINUATION_KWARG,
     as_retry_skipped_deployment_ids,
+    deployment_model_id_pin,
     deployments_the_caller_may_use,
     filter_team_based_models,
     filter_web_search_deployments,
@@ -5317,6 +5318,9 @@ class Router(DeprecatedRouterMembers):
         ):
             kwargs[STREAM_USAGE_INJECTED_FIELD] = True
         self._update_kwargs_with_default_litellm_params(kwargs=kwargs, metadata_variable_name=metadata_variable_name)
+        pinned_model_id: Final = deployment_model_id_pin(deployment_context)
+        if pinned_model_id is not None:
+            kwargs["model_id"] = pinned_model_id  # rebind-ok: this method's contract is filling the caller's kwargs
         hop_kwargs: Final = _FALLBACK_HOP_ADAPTER.validate_python(kwargs)
         if _is_fallback_hop(hop_kwargs):
             EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt(
@@ -6243,6 +6247,9 @@ class Router(DeprecatedRouterMembers):
                     kwargs[k] = v
                 elif k == "metadata":
                     kwargs[k].update(v)
+            pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data))
+            if pinned_model_id is not None:
+                kwargs["model_id"] = pinned_model_id  # rebind-ok: this call's own kwargs, filled like the defaults
 
             # call via litellm.completion()
             return litellm.text_completion(

litellm/router_utils/common_utils.py

@@ -16,6 +16,7 @@ from litellm.exceptions import BadRequestError
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
 from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
+from litellm.llms.sagemaker.common_utils import deployment_inference_component
 from litellm.types.router import (
     CredentialLiteLLMParams,
     Deployment,
@@ -522,6 +523,16 @@ PROVIDER_SPECIFIC_CAPABILITY_FIELDS: Final = frozenset(ProviderSpecificModelInfo
 _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 
 
+def deployment_model_id_pin(deployment_litellm_params: Mapping[str, object]) -> object | None:
+    clientside_params: Final = deployment_litellm_params.get("configurable_clientside_auth_params")
+    if isinstance(clientside_params, list | tuple) and "model_id" in clientside_params:
+        return None
+    own_model_id: Final = deployment_litellm_params.get("model_id")
+    if own_model_id is not None:
+        return own_model_id
+    return deployment_inference_component(deployment_litellm_params)
+
+
 def is_model_capability_field(name: str) -> bool:
     return name in PROVIDER_SPECIFIC_CAPABILITY_FIELDS or (
         name.startswith("supports_") and name in GetModelCostMap.loaded_model_cost_map_fields()

litellm/types/llms/gemini.py

@@ -81,6 +81,14 @@ class BidiGenerateContentServerMessage(TypedDict, total=False):
     """Output only. The setup complete message."""
 
 
+class BidiGenerateContentActivityStart(TypedDict):
+    pass
+
+
+class BidiGenerateContentActivityEnd(TypedDict):
+    pass
+
+
 class BidiGenerateContentRealtimeInput(TypedDict, total=False):
     text: str
     """The text to be sent to the model."""
@@ -94,11 +102,11 @@ class BidiGenerateContentRealtimeInput(TypedDict, total=False):
     audioStreamEnd: bool
     """Output only. If true, indicates that the audio stream has ended."""
 
-    activityStart: bool
-    """Output only. If true, indicates that the activity has started."""
+    activityStart: BidiGenerateContentActivityStart
+    """Marks the start of user activity when automatic activity detection is disabled."""
 
-    activityEnd: bool
-    """Output only. If true, indicates that the activity has ended."""
+    activityEnd: BidiGenerateContentActivityEnd
+    """Marks the end of user activity when automatic activity detection is disabled."""
 
 
 StartOfSpeechSensitivityEnum = Literal[

litellm/types/proxy/management_endpoints/key_requests.py

@@ -1,14 +1,14 @@
 import enum
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
 from datetime import datetime
 from types import MappingProxyType
-from typing import Any, Literal  # noqa: TID251  # allowed_routes and metadata stay Any for existing callers
+from typing import Any, Final, Literal  # noqa: TID251  # allowed_routes and metadata stay Any for existing callers
 
 from pydantic import Field, PositiveInt, field_validator, model_validator
 from typing_extensions import TypeIs  # noqa: TID251  # narrows the raw payload a before-validator receives
 
 from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase
-from litellm.types.router import UpdateRouterConfig
+from litellm.types.router import ContentPolicyFallbackRules, UpdateRouterConfig
 from litellm.types.utils import LiteLLMPydanticObjectBase
 
 
@@ -17,6 +17,12 @@ class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase):
     index_permissions: list[Literal["read", "write"]]
 
 
+def _fallback_targets(rules: ContentPolicyFallbackRules) -> Iterator[object]:
+    for rule in rules:
+        for targets in rule.values():
+            yield from targets
+
+
 class KeyRequestBase(GenerateRequestBase):
     key: str | None = None
     tpd_limit: int | None = None
@@ -42,6 +48,18 @@ class KeyRequestBase(GenerateRequestBase):
     router_settings: UpdateRouterConfig | None = None
     access_group_ids: list[str] | None = None
 
+    @field_validator("router_settings")
+    @classmethod
+    def content_policy_fallbacks_name_models(
+        cls, router_settings: UpdateRouterConfig | None
+    ) -> UpdateRouterConfig | None:
+        rules: Final = None if router_settings is None else router_settings.content_policy_fallbacks
+        if rules is not None and not all(isinstance(target, str) for target in _fallback_targets(rules)):
+            raise ValueError(
+                'A key\'s content_policy_fallbacks targets must be model names, e.g. [{"gpt-4": ["gpt-4o"]}]'
+            )
+        return router_settings
+
 
 class LiteLLMKeyType(str, enum.Enum):
     """

litellm/types/proxy/management_endpoints/subscription_accounts.py

@@ -0,0 +1,62 @@
+from datetime import date
+from typing import Final
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
+
+CURRENCY_CODE_PATTERN: Final = r"^[A-Z]{3}$"
+
+
+class SubscriptionAccountFeeRequest(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    custom_llm_provider: str = Field(min_length=1)
+    account_id: str = Field(min_length=1)
+    monthly_fee: float = Field(gt=0, allow_inf_nan=False)
+    currency: str = Field(pattern=CURRENCY_CODE_PATTERN)
+    billing_period_start: date
+    label: str | None = None
+
+
+class UpdateSubscriptionAccountFeeRequest(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    subscription_account_id: str
+    monthly_fee: float | None = Field(default=None, gt=0, allow_inf_nan=False)
+    currency: str | None = Field(default=None, pattern=CURRENCY_CODE_PATTERN)
+    billing_period_start: date | None = None
+    label: str | None = None
+
+
+class DeleteSubscriptionAccountRequest(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    subscription_account_id: str
+
+
+class BillingPeriod(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    start: date
+    end: date
+
+
+class SubscriptionAccountUsage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    custom_llm_provider: str
+    account_id: str | None
+    deployments: tuple[str, ...]
+    fee: LiteLLM_SubscriptionAccountTable | None
+    billing_periods: tuple[BillingPeriod, ...]
+    fixed_cost: float | None
+
+
+class SubscriptionUsageResponse(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    start_date: date
+    end_date: date
+    subscription_providers: tuple[str, ...]
+    accounts: tuple[SubscriptionAccountUsage, ...]

litellm/types/router.py

@@ -192,6 +192,11 @@ OptionalPreCallChecks = list[
 ]
 
 
+FallbackTargets: TypeAlias = Sequence[str | Mapping[str, JsonValue]]
+ContentPolicyFallbackRule: TypeAlias = Annotated[Mapping[str, FallbackTargets], Field(min_length=1, max_length=1)]
+ContentPolicyFallbackRules: TypeAlias = Sequence[ContentPolicyFallbackRule]
+
+
 class UpdateRouterConfig(LiteLLMBaseModel):
     """
     Set of params that you can modify via `router.update_settings()`.
@@ -211,6 +216,7 @@ class UpdateRouterConfig(LiteLLMBaseModel):
     retry_after: float | None = None
     fallbacks: list[dict] | None = None
     context_window_fallbacks: list[dict] | None = None
+    content_policy_fallbacks: ContentPolicyFallbackRules | None = None
     model_group_alias: dict[str, str | dict] | None = Field(default={})
     enable_tag_filtering: bool | None = None
     weights: RouterWeights | None = None

litellm/types/utils.py

@@ -7,6 +7,7 @@ from enum import Enum
 from types import MappingProxyType
 from typing import (
     TYPE_CHECKING,
+    Annotated,
     Any,
     Final,
     Literal,
@@ -38,6 +39,7 @@ from openai.types.moderation_create_response import (
 )
 from pydantic import (
     BaseModel,
+    BeforeValidator,
     ConfigDict,
     Field,
     FieldSerializationInfo,
@@ -45,6 +47,7 @@ from pydantic import (
     JsonValue,
     PrivateAttr,
     SkipValidation,
+    TypeAdapter,
     ValidationError,
     field_serializer,
     field_validator,
@@ -1199,7 +1202,7 @@ class ChatCompletionTokenLogprob(OpenAIObject):
     unlikely.
     """
 
-    top_logprobs: list[TopLogprob]
+    top_logprobs: list[TopLogprob] = Field(default_factory=list)
     """List of the most likely tokens and their log probability, at this token
     position.
 
@@ -1236,9 +1239,13 @@ class ChatCompletionTokenLogprob(OpenAIObject):
 
 
 class ChoiceLogprobs(OpenAIObject):
+    model_config = ConfigDict(defer_build=False)
+
     content: list[ChatCompletionTokenLogprob] | None = None
     """A list of message content tokens with log probability information."""
 
+    refusal: Sequence[ChatCompletionTokenLogprob] | None = None
+
     def __contains__(self, key) -> bool:
         # Define custom behavior for the 'in' operator
         return hasattr(self, key)
@@ -1252,6 +1259,21 @@ class ChoiceLogprobs(OpenAIObject):
         return getattr(self, key)
 
 
+def _sdk_logprobs_as_dict(logprobs: object) -> object:
+    if isinstance(logprobs, ChoiceLogprobs) or not isinstance(logprobs, BaseModel):
+        return logprobs
+    return logprobs.model_dump(exclude_unset=True)
+
+
+CHOICE_LOGPROBS_OR_JSON: Final[TypeAdapter[ChoiceLogprobs | JsonValue]] = TypeAdapter(
+    Annotated[
+        ChoiceLogprobs | JsonValue,
+        Field(union_mode="left_to_right"),
+        BeforeValidator(_sdk_logprobs_as_dict),
+    ]
+)
+
+
 class FunctionCall(OpenAIObject):
     arguments: str
     name: str | None = None
@@ -1765,7 +1787,7 @@ class Choices(SafeAttributeModel, OpenAIObject):
                 params["message"] = Message(**dump)
         if logprobs is not None:
             if isinstance(logprobs, dict):
-                params["logprobs"] = ChoiceLogprobs(**logprobs)
+                params["logprobs"] = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs)
             else:
                 params["logprobs"] = logprobs
         else:
@@ -2131,8 +2153,8 @@ class StreamingChoices(OpenAIObject):
         if enhancements is not None:
             self.enhancements = enhancements
 
-        if logprobs is not None and isinstance(logprobs, dict):
-            self.logprobs = ChoiceLogprobs(**logprobs)
+        if isinstance(logprobs, (dict, BaseModel)):
+            self.logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs)
         else:
             self.logprobs = logprobs
 
@@ -4155,6 +4177,8 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
         "supports_assistant_prefill",
         "supports_bedrock_native_structured_output",
         "supports_pdf_input",
+        "reasoning_modes",
+        "reasoning_contexts",
     }
 )
 

litellm/utils.py

@@ -10956,7 +10956,7 @@ class ProviderConfigManager:
         if LlmProviders.GEMINI == provider:
             from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
 
-            return GeminiRealtimeConfig()
+            return GeminiRealtimeConfig(drop_params=drop_params)
         if LlmProviders.META == provider:
             from litellm.llms.meta.realtime.transformation import MetaRealtimeConfig
 

schema.prisma

@@ -2131,3 +2131,19 @@ model LiteLLM_AccountingDelivery {
   @@id([event_id, target, version])
   @@index([target, state, next_attempt_at])
 }
+
+model LiteLLM_SubscriptionAccountTable {
+  subscription_account_id String @id @default(uuid())
+  custom_llm_provider String
+  account_id String
+  label String?
+  monthly_fee Float
+  currency String
+  billing_period_start String
+  created_at    DateTime               @default(now()) @map("created_at")
+  created_by String
+  updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")
+  updated_by String
+
+  @@unique([custom_llm_provider, account_id], map: "LiteLLM_SubscriptionAccountTable_provider_account_key")
+}

tests/benchmarks/test_realtime_session_merge_benchmarks.py

@@ -0,0 +1,21 @@
+from typing import Final
+
+import pytest
+
+from litellm.litellm_core_utils.realtime_streaming import _merge_session_fields
+
+NESTED_LEVELS: Final = 8
+LEAVES: Final = 2000
+
+
+def _nested_session(depth: int) -> dict[str, object]:
+    return {"child": _nested_session(depth - 1)} if depth else {f"k{index}": index for index in range(LEAVES)}
+
+
+@pytest.mark.benchmark
+def test_merging_a_session_update_that_repeats_every_nested_level() -> None:
+    session: Final = _nested_session(NESTED_LEVELS)
+
+    merged: Final = _merge_session_fields(session, session)
+
+    assert merged == session

tests/code_coverage_tests/recursive_detector.py

@@ -85,6 +85,7 @@ IGNORE_FUNCTIONS = [
     "_sort_processed_sets",  # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
     "scrub_json_strings",  # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap.
     "_without_requester_ip_address",  # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did.
+    "_merge_session_fields",  # max depth set (_SESSION_MERGE_MAX_DEPTH); deeper mappings are replaced wholesale by the later session.update.
 ]
 
 

tests/e2e/llm_translation/realtime/realtime_client.py

@@ -47,6 +47,7 @@ class RealtimeProvider:
     id: str
     alias: str
     litellm_params: LiteLLMParamsBody
+    modalities: tuple[str, ...] = ("text",)
 
 
 PROVIDERS = (
@@ -75,6 +76,7 @@ PROVIDERS = (
             model="gemini/gemini-3.1-flash-live-preview",
             api_key="os.environ/GEMINI_API_KEY",
         ),
+        modalities=("audio",),
     ),
     RealtimeProvider(
         "vertex_ai",
@@ -84,6 +86,7 @@ PROVIDERS = (
             vertex_location="us-central1",
             vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
         ),
+        modalities=("audio",),
     ),
     RealtimeProvider(
         "bedrock",

tests/e2e/llm_translation/realtime/test_realtime_e2e.py

@@ -124,7 +124,8 @@ def test_text_conversation(
         session.send(
             SessionUpdate(
                 session=SessionConfig(
-                    instructions="You are a terse assistant. Reply in one short sentence."
+                    modalities=list(provider.modalities),
+                    instructions="You are a terse assistant. Reply in one short sentence.",
                 )
             )
         )
@@ -159,6 +160,7 @@ def test_tool_call_round_trip(
         session.send(
             SessionUpdate(
                 session=SessionConfig(
+                    modalities=list(provider.modalities),
                     tools=[WEATHER_TOOL],
                     tool_choice="auto",
                     instructions=(

tests/integration/authorization/test_claude_code_gateway_oauth.py

@@ -636,3 +636,55 @@ def test_device_flow_mints_the_session_for_the_team_the_developer_picks_in_the_b
             served_models: Final = {model.id for model in sdk.models.list()}
         assert served_models == {team_models[chosen_team]}
         assert len(identity_provider.drain()) == 3
+
+
+@pytest.mark.timeout(300)
+def test_gateway_login_lists_and_defaults_to_the_first_team_on_the_user_record_after_a_write_to_its_row(
+    enabled_gateway: Gateway, identity_provider: Wire
+) -> None:
+    run_id: Final = uuid.uuid4().hex
+    with enabled_gateway.scenario() as scenario:
+        user_id: Final = scenario.user(user_id=_USER_ID, user_email=_USER_EMAIL, user_role="internal_user")
+        first_team: Final = scenario.team(team_id=f"zeta-{run_id}")
+        second_team: Final = scenario.team(team_id=f"alpha-{run_id}")
+        for team_id in (first_team, second_team):
+            enabled_gateway.post(
+                "/team/member_add", {"team_id": team_id, "member": {"role": "user", "user_id": user_id}}
+            )
+        enabled_gateway.post("/team/update", {"team_id": first_team, "metadata": {"updated_by_run": run_id}})
+        assert enabled_gateway.rows('SELECT teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (user_id,)) == [
+            {"teams": [first_team, second_team]}
+        ]
+
+        authorization_response: Final = enabled_gateway.client.post(
+            "/claude_code_gateway/oauth/device_authorization", data={"client_id": _CLIENT_ID}
+        )
+        assert authorization_response.status_code == 200, authorization_response.text
+        authorization: Final = _DeviceAuthorization.model_validate_json(authorization_response.content)
+        verification: Final = enabled_gateway.client.get(authorization.verification_uri, follow_redirects=False)
+        assert verification.status_code == 303, verification.text
+        identity_provider_redirect: Final = enabled_gateway.client.get(
+            verification.headers["location"], follow_redirects=False
+        )
+        assert identity_provider_redirect.status_code == 302, identity_provider_redirect.text
+        callback: Final = enabled_gateway.client.get(
+            identity_provider_redirect.headers["location"], follow_redirects=False
+        )
+        assert callback.status_code == 200, callback.text
+        assert tuple(_TEAM_OPTION.findall(callback.text)) == (first_team, second_team), callback.text
+        browser_token: Final = _BROWSER_TOKEN.search(callback.text)
+        assert browser_token is not None, callback.text
+        completion_without_a_pick: Final = enabled_gateway.client.post(
+            f"/sso/cli/complete/{authorization.device_code.partition('.')[0]}",
+            data={
+                "user_code": authorization.user_code,
+                "browser_complete_token": html.unescape(browser_token.group(1)),
+            },
+        )
+        assert completion_without_a_pick.status_code == 200, completion_without_a_pick.text
+        token_response: Final = _device_code_grant(enabled_gateway, authorization.device_code)
+        assert token_response.status_code == 200, token_response.text
+        token: Final = _AccessToken.model_validate_json(token_response.content)
+        identity_provider.drain()
+
+        assert (token.user_id, token.team_id) == (user_id, first_team)

tests/integration/management/test_added_backup_deployment_serves_peer_fallbacks.py

@@ -0,0 +1,109 @@
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Final
+
+import httpx
+import pytest
+from pydantic import TypeAdapter
+from redis import Redis
+
+from tests.integration._support.client import Gateway, eventually, list_value, object_value, string_value
+from tests.integration._support.database import scratch_database
+from tests.integration._support.process import graceful_stop_seconds, owned_proxy
+from tests.integration._support.redis_process import owned_redis
+
+pytestmark: Final = [pytest.mark.timeout(int(4 * graceful_stop_seconds() + 120))]
+
+COORDINATION_CONFIG: Final = Path(__file__).resolve().parents[1] / "coordination_redis_proxy_config.yaml"
+CONFIG_SYNC_CHANNEL: Final = "litellm_proxy.config_change"
+SUBSCRIBER_COUNTS: Final = TypeAdapter(tuple[tuple[str, int], ...])
+PERIODIC_RELOAD_SECONDS: Final = 3600
+PEER_PICKUP_SECONDS: Final = 4
+PROVIDER_KEY: Final = "integration-provider-key"
+
+
+@contextmanager
+def _proxy_pair(gateway: Gateway, directory: Path) -> Iterator[tuple[Gateway, Gateway]]:
+    with scratch_database("integration_model_add_sync") as database, owned_redis(directory) as coordination:
+        shared_stores: Final = {
+            "DATABASE_URL": database,
+            "REDIS_HOST": coordination.host,
+            "REDIS_PORT": str(coordination.port),
+            "PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": str(PERIODIC_RELOAD_SECONDS),
+        }
+        with (
+            owned_proxy(gateway, directory / "writer", shared_stores, config=COORDINATION_CONFIG) as writer,
+            owned_proxy(gateway, directory / "peer", shared_stores, config=COORDINATION_CONFIG) as peer,
+            Redis(host=coordination.host, port=coordination.port, socket_timeout=2) as redis,
+        ):
+            eventually(
+                lambda: SUBSCRIBER_COUNTS.validate_python(redis.pubsub_numsub(CONFIG_SYNC_CHANNEL))[0][1],
+                lambda count: count >= 2,
+                seconds=30,
+            )
+            yield writer, peer
+
+
+@contextmanager
+def _failing_upstream_model(gateway: Gateway, upstream_model: str) -> Iterator[None]:
+    script: Final = f"{gateway.upstream_url}/__scripts/{upstream_model}"
+    httpx.post(script, json={"statuses": [500]}, timeout=5, trust_env=False).raise_for_status()
+    try:
+        yield
+    finally:
+        httpx.delete(script, timeout=5, trust_env=False).raise_for_status()
+
+
+def _add_deployment(writer: Gateway, group: str, upstream_model: str, order: int) -> None:
+    writer.post(
+        "/model/new",
+        {
+            "model_name": group,
+            "litellm_params": {
+                "model": f"openai/{upstream_model}",
+                "api_key": PROVIDER_KEY,
+                "api_base": f"{writer.upstream_url}/v1",
+                "order": order,
+                "num_retries": 0,
+            },
+        },
+    )
+
+
+def _listed_models(proxy: Gateway) -> frozenset[str]:
+    return frozenset(string_value(object_value(entry)["id"]) for entry in list_value(proxy.get("/v1/models")["data"]))
+
+
+def _chat(proxy: Gateway, group: str) -> httpx.Response:
+    return proxy.request(
+        "POST",
+        "/v1/chat/completions",
+        {"model": group, "messages": [{"role": "user", "content": f"order fallback {uuid.uuid4().hex}"}]},
+    )
+
+
+def test_a_backup_added_on_one_proxy_catches_the_peers_failing_primary_right_away(
+    gateway: Gateway, tmp_path: Path
+) -> None:
+    group: Final = f"integration-{uuid.uuid4().hex}"
+    primary: Final = f"integration-primary-{uuid.uuid4().hex}"
+    backup: Final = f"integration-backup-{uuid.uuid4().hex}"
+    with _proxy_pair(gateway, tmp_path) as (writer, peer), _failing_upstream_model(gateway, primary):
+        _add_deployment(writer, group, primary, order=1)
+        eventually(lambda: _listed_models(peer), lambda listed: group in listed, seconds=30)
+
+        _add_deployment(writer, group, backup, order=2)
+        on_writer: Final = _chat(writer, group)
+        on_peer: Final = eventually(
+            lambda: _chat(peer, group),
+            lambda response: response.status_code == 200,
+            seconds=PEER_PICKUP_SECONDS,
+            return_last_on_timeout=True,
+        )
+
+    assert on_writer.status_code == 200, on_writer.text
+    assert object_value(on_writer.json())["model"] == backup, on_writer.text
+    assert on_peer.status_code == 200, on_peer.text
+    assert object_value(on_peer.json())["model"] == backup, on_peer.text

tests/integration/providers/test_realtime_transcription_guardrail_session_update.py

@@ -355,7 +355,13 @@ def _voice_scenario_created_after_the_first_frame(transcript: str) -> RealtimeRe
 
 
 def _live_scenario() -> RealtimeResponse:
-    return RealtimeResponse(content_type="application/x-realtime", events=())
+    return RealtimeResponse(
+        content_type="application/x-realtime",
+        events=(
+            {"serverContent": {"modelTurn": {"parts": [{"text": "scripted $REQUEST_ID"}]}}},
+            {"serverContent": {"turnComplete": True}},
+        ),
+    )
 
 
 def _realtime_update(turn_detection: JsonValue) -> dict[str, JsonValue]:
@@ -1348,10 +1354,14 @@ def test_gemini_ai_studio_voice_session_with_vad_tuning_connects_under_the_guard
             _ws_base(_owned_url(guardrail_proxy)),
             f"model={model}",
             key,
-            (_step(_update_frame(update), until="session.updated"),),
+            (
+                _step(_update_frame(update), until="session.updated"),
+                _step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE),
+            ),
         )
         observed: Final = _observed(upstream_url, handle.scenario_id)
-        assert session.types == ("session.created", "session.updated"), session
+        assert session.types[:2] == ("session.created", "session.updated"), session
+        assert session.types[-1] == RESPONSE_DONE, session
         assert _setup_detection(observed) == {"disabled": True}, _sent(observed)
 
 
@@ -1365,10 +1375,14 @@ def test_vertex_live_voice_session_keeps_vad_tuning_under_the_guardrail(guardrai
             _ws_base(_owned_url(guardrail_proxy)),
             f"model={model}",
             key,
-            (_step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),),
+            (
+                _step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),
+                _step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE),
+            ),
         )
         observed: Final = _observed(upstream_url, handle.scenario_id)
-        assert session.types == ("session.created", "session.updated"), session
+        assert session.types[:2] == ("session.created", "session.updated"), session
+        assert session.types[-1] == RESPONSE_DONE, session
         assert _setup_detection(observed) == GEMINI_TUNED_DETECTION, _sent(observed)
 
 

tests/integration/providers/test_sagemaker_chat_wire.py

@@ -243,7 +243,7 @@ def test_sagemaker_chat_signs_the_inference_component_header_and_sends_hf_model_
     )
     assert "x-amzn-sagemaker-inference-component" in signed_headers.removeprefix("SignedHeaders=").split(";")
     assert JSON_OBJECT.validate_json(request.body) == {
-        **container_body(wire.url, PROMPT, stream=False, model_id=_INFERENCE_COMPONENT),
+        **container_body(wire.url, PROMPT, stream=False),
         "model": SERVED_MODEL,
     }, request.body
 

tests/integration/providers/test_sagemaker_embedding_wire.py

@@ -668,12 +668,12 @@ def test_an_extra_body_that_is_not_a_populated_object_adds_nothing_to_the_body(
     assert _body(request) == {"input": [text]}, request.body
 
 
-def test_a_caller_sent_model_id_overrides_the_deployments_inference_component(proxy: Gateway, wire: Wire) -> None:
+def test_a_caller_sent_model_id_cannot_override_the_deployments_inference_component(proxy: Gateway, wire: Wire) -> None:
     text: Final = _text()
     response: Final = _embed(proxy, "sm-openai", [text], model_id="caller-chosen-component")
     assert response.status_code == 200, response.text
     request: Final = _only_request(wire)
-    assert request.headers[COMPONENT_HEADER] == "caller-chosen-component", dict(request.headers)
+    assert request.headers[COMPONENT_HEADER] == COMPONENT, dict(request.headers)
     assert _body(request) == {"input": [text], "model": SERVED_MODEL}, request.body
 
 

tests/integration/spend/test_chatgpt_subscription_usage.py

@@ -0,0 +1,322 @@
+import uuid
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Final
+
+import httpx
+import pytest
+from integration._support import codex_vendor as cv
+from integration._support import responses_vendor as rv
+from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, string_value
+from integration._support.process import OwnedProxy, owned_proxy_process
+from integration._support.wire import Wire, wire_server
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
+
+_MODEL: Final = "chatgpt/gpt-5.5"
+_PROVIDER: Final = "chatgpt"
+_FEE_TABLE_QUERY: Final = (
+    'SELECT subscription_account_id FROM "LiteLLM_SubscriptionAccountTable" WHERE custom_llm_provider = %s'
+    " AND account_id = %s"
+)
+_FEE_ROW_QUERY: Final = (
+    'SELECT subscription_account_id FROM "LiteLLM_SubscriptionAccountTable" WHERE subscription_account_id = %s'
+)
+_MONTHLY_FEE: Final = 200.0
+_BILLING_PERIOD_START: Final = "2026-01-15"
+_RANGE: Final = {"start_date": "2026-03-01", "end_date": "2026-03-31"}
+_PERIOD_IN_RANGE: Final = ("2026-03-15", "2026-04-14")
+_TOKENS: Final = TypeAdapter(int)
+
+
+class _Metrics(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    spend: float
+    prompt_tokens: int
+    completion_tokens: int
+    api_requests: int
+    successful_requests: int
+    failed_requests: int
+
+
+class _ProviderUsage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    metrics: _Metrics
+    metadata: Mapping[str, JsonValue]
+
+
+class _Breakdown(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    providers: Mapping[str, _ProviderUsage]
+
+
+class _Day(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    breakdown: _Breakdown
+
+
+class _DailyActivity(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    results: tuple[_Day, ...]
+
+
+class _Fee(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    subscription_account_id: str
+    custom_llm_provider: str
+    account_id: str
+    monthly_fee: float
+    currency: str
+    billing_period_start: str
+
+
+class _Period(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    start: str
+    end: str
+
+
+class _Account(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    custom_llm_provider: str
+    account_id: str | None
+    deployments: tuple[str, ...]
+    fee: _Fee | None
+    billing_periods: tuple[_Period, ...]
+    fixed_cost: float | None
+
+
+class _SubscriptionUsage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    accounts: tuple[_Account, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class _Rig:
+    wire: Wire
+    proxy: OwnedProxy
+    model: str
+
+    @property
+    def gateway(self) -> Gateway:
+        return self.proxy.gateway
+
+
+@pytest.fixture(scope="module")
+def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
+    directory: Final = tmp_path_factory.mktemp("chatgpt-subscription-usage")
+    no_default_login: Final = directory / "no-default-login"
+    no_default_login.mkdir()
+    deployment_login: Final = cv.login(directory) / "auth.json"
+    with gateway_from_environment() as gateway, wire_server(cv.CodexVendor().respond) as wire:
+        overrides: Final = {"CHATGPT_TOKEN_DIR": str(no_default_login), "CHATGPT_API_BASE": wire.url}
+        config: Final = cv.proxy_config(directory, probe=False)
+        with (
+            owned_proxy_process(gateway, directory, overrides, config=config, workers=2) as owned,
+            owned.gateway.scenario() as scenario,
+        ):
+            model: Final = scenario.model(
+                model=_MODEL, api_base=wire.url, api_key=None, chatgpt_auth_file=str(deployment_login)
+            )
+            yield _Rig(wire, owned, model)
+
+
+def _responses_body(model: str, marker: str, *, stream: bool) -> Mapping[str, JsonValue]:
+    return {
+        "model": model,
+        "input": [{"role": "user", "content": f"Say marker-{marker}"}],
+        "stream": stream,
+        "cache": {"no-cache": True},
+    }
+
+
+def _assert_answered(response: httpx.Response, marker: str, content_type: str) -> None:
+    assert response.status_code == 200, response.text
+    assert response.headers["content-type"].startswith(content_type), response.headers
+    assert rv.answer(marker) in response.text, response.text
+
+
+def _today() -> str:
+    return datetime.now(UTC).date().isoformat()
+
+
+def _chatgpt_usage_between(gateway: Gateway, first_day: str, last_day: str) -> tuple[_ProviderUsage, ...]:
+    response: Final = gateway.request(
+        "GET", "/user/daily/activity", params={"start_date": first_day, "end_date": last_day}
+    )
+    assert response.status_code == 200, response.text
+    activity: Final = _DailyActivity.model_validate_json(response.content)
+    return tuple(
+        result.breakdown.providers[_PROVIDER] for result in activity.results if _PROVIDER in result.breakdown.providers
+    )
+
+
+def _summed(buckets: tuple[_ProviderUsage, ...]) -> _Metrics:
+    return _Metrics(
+        spend=sum(bucket.metrics.spend for bucket in buckets),
+        prompt_tokens=sum(bucket.metrics.prompt_tokens for bucket in buckets),
+        completion_tokens=sum(bucket.metrics.completion_tokens for bucket in buckets),
+        api_requests=sum(bucket.metrics.api_requests for bucket in buckets),
+        successful_requests=sum(bucket.metrics.successful_requests for bucket in buckets),
+        failed_requests=sum(bucket.metrics.failed_requests for bucket in buckets),
+    )
+
+
+def _usage(gateway: Gateway, *, key: str | None = None) -> httpx.Response:
+    return gateway.request("GET", "/subscription_accounts/usage", params=_RANGE, key=key)
+
+
+def _accounts(gateway: Gateway) -> tuple[_Account, ...]:
+    response: Final = _usage(gateway)
+    assert response.status_code == 200, response.text
+    return _SubscriptionUsage.model_validate_json(response.content).accounts
+
+
+def _deployment_account(rig: _Rig) -> _Account:
+    (account,) = _accounts(rig.gateway)
+    assert (account.custom_llm_provider, account.account_id, account.deployments) == (
+        _PROVIDER,
+        cv.ACCOUNT,
+        (rig.model,),
+    ), account
+    return account
+
+
+def _assert_no_fee(account: _Account) -> None:
+    assert (account.fee, account.billing_periods, account.fixed_cost) == (None, (), None), account
+
+
+def _fee_body(account_id: str, monthly_fee: float = _MONTHLY_FEE) -> Mapping[str, JsonValue]:
+    return {
+        "custom_llm_provider": _PROVIDER,
+        "account_id": account_id,
+        "monthly_fee": monthly_fee,
+        "currency": "USD",
+        "billing_period_start": _BILLING_PERIOD_START,
+        "label": "integration",
+    }
+
+
+def _delete_fee_if_present(gateway: Gateway, identity: str) -> None:
+    if gateway.rows(_FEE_ROW_QUERY, (identity,)):
+        gateway.post("/subscription_accounts/delete", {"subscription_account_id": identity})
+    assert gateway.rows(_FEE_ROW_QUERY, (identity,)) == []
+
+
+def _set_fee(scenario: Scenario, account_id: str) -> str:
+    created: Final = scenario.gateway.post("/subscription_accounts/new", _fee_body(account_id))
+    identity: Final = string_value(created["subscription_account_id"])
+    scenario.cleanups.callback(_delete_fee_if_present, scenario.gateway, identity)
+    return identity
+
+
+def _assert_fee_reported(account: _Account, identity: str) -> None:
+    assert account.fee is not None, account
+    assert (
+        account.fee.subscription_account_id,
+        account.fee.account_id,
+        account.fee.monthly_fee,
+        account.fee.currency,
+        account.fee.billing_period_start,
+    ) == (identity, cv.ACCOUNT, _MONTHLY_FEE, "USD", _BILLING_PERIOD_START), account
+    assert tuple((period.start, period.end) for period in account.billing_periods) == (_PERIOD_IN_RANGE,), account
+    assert account.fixed_cost == _MONTHLY_FEE, account
+
+
+def test_responses_calls_on_a_deployment_login_count_as_subscription_covered_chatgpt_usage(rig: _Rig) -> None:
+    first_day: Final = _today()
+    rig.wire.drain()
+    plain_marker: Final = uuid.uuid4().hex
+    stream_marker: Final = uuid.uuid4().hex
+    plain: Final = rig.gateway.request("POST", "/v1/responses", _responses_body(rig.model, plain_marker, stream=False))
+    _assert_answered(plain, plain_marker, "application/json")
+    streamed: Final = rig.gateway.request(
+        "POST", "/v1/responses", _responses_body(rig.model, stream_marker, stream=True)
+    )
+    _assert_answered(streamed, stream_marker, "text/event-stream")
+    forwarded: Final = rig.wire.drain()
+    assert len(forwarded) == 2, [request.target for request in forwarded]
+    for request, marker in zip(forwarded, (plain_marker, stream_marker), strict=True):
+        cv.forwarded(request, marker)
+    last_day: Final = _today()
+    buckets: Final = eventually(
+        lambda: _chatgpt_usage_between(rig.gateway, first_day, last_day),
+        lambda found: _summed(found).api_requests >= 2,
+        seconds=70,
+    )
+    totals: Final = _summed(buckets)
+    assert (totals.api_requests, totals.successful_requests, totals.failed_requests, totals.spend) == (
+        2,
+        2,
+        0,
+        0,
+    ), buckets
+    assert (totals.prompt_tokens, totals.completion_tokens) == (
+        2 * _TOKENS.validate_python(cv.USAGE["input_tokens"]),
+        2 * _TOKENS.validate_python(cv.USAGE["output_tokens"]),
+    ), buckets
+    assert all(bucket.metadata.get("subscription_covered") is True for bucket in buckets), buckets
+
+
+def test_fee_on_the_deployment_account_reports_its_fixed_cost_until_deleted(rig: _Rig) -> None:
+    _assert_no_fee(_deployment_account(rig))
+    with rig.gateway.scenario() as scenario:
+        identity: Final = _set_fee(scenario, cv.ACCOUNT)
+        _assert_fee_reported(_deployment_account(rig), identity)
+        deleted: Final = rig.gateway.post("/subscription_accounts/delete", {"subscription_account_id": identity})
+        assert deleted["subscription_account_id"] == identity, deleted
+        assert rig.gateway.rows(_FEE_ROW_QUERY, (identity,)) == []
+        _assert_no_fee(_deployment_account(rig))
+
+
+def test_second_fee_for_the_same_account_answers_400_and_keeps_the_first(rig: _Rig) -> None:
+    with rig.gateway.scenario() as scenario:
+        identity: Final = _set_fee(scenario, cv.ACCOUNT)
+        duplicate: Final = rig.gateway.request(
+            "POST", "/subscription_accounts/new", _fee_body(cv.ACCOUNT, monthly_fee=3 * _MONTHLY_FEE)
+        )
+        assert duplicate.status_code == 400, duplicate.text
+        assert rig.gateway.rows(_FEE_TABLE_QUERY, (_PROVIDER, cv.ACCOUNT)) == [{"subscription_account_id": identity}]
+        _assert_fee_reported(_deployment_account(rig), identity)
+
+
+def test_update_and_delete_of_an_unknown_fee_answer_404(rig: _Rig) -> None:
+    unknown: Final = str(uuid.uuid4())
+    updated: Final = rig.gateway.request(
+        "POST", "/subscription_accounts/update", {"subscription_account_id": unknown, "monthly_fee": _MONTHLY_FEE}
+    )
+    assert updated.status_code == 404, updated.text
+    deleted: Final = rig.gateway.request("POST", "/subscription_accounts/delete", {"subscription_account_id": unknown})
+    assert deleted.status_code == 404, deleted.text
+    assert rig.gateway.rows(_FEE_ROW_QUERY, (unknown,)) == []
+
+
+def test_internal_user_is_refused_setting_a_fee_and_reading_usage(rig: _Rig) -> None:
+    with rig.gateway.scenario() as scenario:
+        key: Final = scenario.key(user_id=scenario.user(user_role="internal_user"))
+        created: Final = rig.gateway.request("POST", "/subscription_accounts/new", _fee_body(cv.ACCOUNT), key=key)
+        assert created.status_code == 401, created.text
+        assert rig.gateway.rows(_FEE_TABLE_QUERY, (_PROVIDER, cv.ACCOUNT)) == []
+        usage: Final = _usage(rig.gateway, key=key)
+        assert usage.status_code == 401, usage.text
+
+
+def test_fee_on_an_account_no_deployment_uses_stays_listed_and_counted(rig: _Rig) -> None:
+    unused: Final = f"acct-unused-{uuid.uuid4().hex}"
+    with rig.gateway.scenario() as scenario:
+        identity: Final = _set_fee(scenario, unused)
+        accounts: Final = _accounts(rig.gateway)
+        assert frozenset(account.account_id for account in accounts) == {cv.ACCOUNT, unused}, accounts
+        (unused_account,) = (account for account in accounts if account.account_id == unused)
+        assert unused_account.fee is not None, unused_account
+        assert (unused_account.deployments, unused_account.fee.subscription_account_id) == ((), identity)
+        assert unused_account.fixed_cost == _MONTHLY_FEE, unused_account

tests/unit/integrations/test_anthropic_cache_control_hook.py

@@ -3998,6 +3998,77 @@ class TestChatPathProviderStamp:
         assert params["prompt_cache_options"] == {"mode": "explicit"}
 
 
+class TestChatPathResolvesTheDeploymentProvider:
+    MODEL = "azure_ai/gpt-6-astra"
+    AMBIENT_API_BASE = "https://other-deployment.openai.azure.com"
+    FOUNDRY_API_BASE = "https://foundry.services.ai.azure.com"
+    UPSTREAM_PROXY_API_BASE = "https://upstream-proxy.example.com"
+    POINTS = [{"location": "message", "role": "system"}]
+    MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}]
+
+    def _pin_ambient_api_base(self, monkeypatch: pytest.MonkeyPatch, pinned: bool) -> None:
+        if pinned:
+            monkeypatch.setenv("AZURE_AI_API_BASE", self.AMBIENT_API_BASE)
+        else:
+            monkeypatch.delenv("AZURE_AI_API_BASE", raising=False)
+
+    def _chat(
+        self, custom_llm_provider: str | None, api_base: str, model: str = MODEL
+    ) -> tuple[list[AllMessageValues], dict[str, object]]:
+        seeded: Final[dict[str, object]] = {"cache_control_injection_points": copy.deepcopy(self.POINTS)}
+        AnthropicCacheControlHook.maybe_seed_default_injection_points(
+            non_default_params=seeded,
+            messages=copy.deepcopy(self.MESSAGES),
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            api_base=api_base,
+        )
+        _, out, params = AnthropicCacheControlHook().get_chat_completion_prompt(
+            model=model,
+            messages=copy.deepcopy(self.MESSAGES),
+            non_default_params=seeded,
+            prompt_id=None,
+            prompt_variables=None,
+            dynamic_callback_params={},
+        )
+        return out, params
+
+    def test_chat_path_stamps_the_breakpoint_on_a_foundry_deployment_while_the_ambient_base_is_azure_openai(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        self._pin_ambient_api_base(monkeypatch, True)
+        messages, params = self._chat(None, self.FOUNDRY_API_BASE)
+        assert (messages, params) == self._chat("azure_ai", self.FOUNDRY_API_BASE)
+        assert AnthropicCacheControlHook.count_request_cache_breakpoints(messages) == 1
+        assert ("prompt_cache_options" in params) == supports_openai_prompt_cache_breakpoint(self.MODEL, "azure_ai")
+
+    @pytest.mark.parametrize("pinned", [True, False])
+    @pytest.mark.parametrize(
+        "api_base",
+        [FOUNDRY_API_BASE, "https://resource.cognitiveservices.azure.com", "https://resource.openai.azure.com"],
+    )
+    def test_chat_path_marks_the_deployment_like_the_provider_its_api_base_dispatches_to(
+        self, monkeypatch: pytest.MonkeyPatch, api_base: str, pinned: bool
+    ) -> None:
+        self._pin_ambient_api_base(monkeypatch, pinned)
+        dispatched_provider: Final = litellm.get_llm_provider(model=self.MODEL, api_base=api_base)[1]
+        assert self._chat(None, api_base) == self._chat(dispatched_provider, api_base)
+
+    @pytest.mark.parametrize("model", [MODEL, "bedrock_mantle/openai.gpt-5.6-sol"])
+    @pytest.mark.parametrize("set_globally", ["env", "module"])
+    def test_chat_path_marks_the_deployment_like_litellm_proxy_when_calls_route_to_one(
+        self, monkeypatch: pytest.MonkeyPatch, model: str, set_globally: str
+    ) -> None:
+        self._pin_ambient_api_base(monkeypatch, False)
+        if set_globally == "env":
+            monkeypatch.setenv("USE_LITELLM_PROXY", "True")
+        else:
+            monkeypatch.setattr(litellm, "use_litellm_proxy", True)
+        assert self._chat(None, self.UPSTREAM_PROXY_API_BASE, model) == self._chat(
+            "litellm_proxy", self.UPSTREAM_PROXY_API_BASE, model
+        )
+
+
 class TestClientBreakpointsCountedOnce:
     def test_client_message_breakpoints_are_not_double_counted(self):
         messages = [

tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py

@@ -1,6 +1,7 @@
 import asyncio
 import importlib
 import json
+from collections.abc import Mapping
 from datetime import datetime, timedelta
 from typing import Final
 
@@ -1111,6 +1112,44 @@ def test_convert_to_model_response_object_with_logprobs():
     assert result.system_fingerprint is None
     assert result._hidden_params is not None
 
+
+@pytest.mark.parametrize(
+    "upstream_logprobs",
+    [
+        pytest.param({"content": [{"token": "Hi", "logprob": None}]}, id="null-content-logprob"),
+        pytest.param(
+            {
+                "content": [{"token": "Hi", "logprob": -0.1, "top_logprobs": []}],
+                "refusal": [{"token": "I", "logprob": None}],
+            },
+            id="null-refusal-logprob",
+        ),
+    ],
+)
+def test_convert_to_model_response_object_passes_nonconforming_logprobs_through(
+    upstream_logprobs: Mapping[str, object],
+) -> None:
+    result: Final = convert_to_model_response_object(
+        response_object={
+            "id": "chatcmpl-123",
+            "object": "chat.completion",
+            "created": 1702685778,
+            "model": "test-model",
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {"role": "assistant", "content": "Hi"},
+                    "logprobs": upstream_logprobs,
+                    "finish_reason": "stop",
+                }
+            ],
+        },
+        model_response_object=ModelResponse(),
+    )
+
+    assert json.loads(result.model_dump_json())["choices"][0]["logprobs"] == upstream_logprobs
+
+
 def test_convert_to_model_response_object_error():
     """Test error handling for None response object."""
     with pytest.raises(Exception, match="Error in response object format"):

tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py

@@ -1246,6 +1246,80 @@ def test_anthropic_messages_pt_keeps_signed_thinking_block():
     assert thinking_blocks[0]["thinking"] == "genuine anthropic reasoning"
 
 
+@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai"))
+def test_anthropic_messages_pt_converts_pdf_image_url_data_uri_to_document(llm_provider: str) -> None:
+    pdf_data = "JVBERi0xLjQK"
+    result: Final = anthropic_messages_pt(
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "summarize"},
+                    {"type": "image_url", "image_url": {"url": f"data:application/pdf;base64,{pdf_data}"}},
+                ],
+            }
+        ],
+        model="claude-opus-5-5",
+        llm_provider=llm_provider,
+    )
+
+    assert result[0]["content"][1] == {
+        "type": "document",
+        "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data},
+    }
+
+
+@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai"))
+def test_anthropic_messages_pt_uses_pdf_format_for_generic_mime_data_uri(llm_provider: str) -> None:
+    pdf_data = "JVBERi0xLjQK"
+    result: Final = anthropic_messages_pt(
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "summarize"},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:application/octet-stream;base64,{pdf_data}",
+                            "format": "application/pdf",
+                        },
+                    },
+                ],
+            }
+        ],
+        model="claude-opus-5-5",
+        llm_provider=llm_provider,
+    )
+
+    assert result[0]["content"][1] == {
+        "type": "document",
+        "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data},
+    }
+
+
+def test_anthropic_messages_pt_keeps_png_image_url_as_image() -> None:
+    image_data = "iVBORw0KGgo="
+    result: Final = anthropic_messages_pt(
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "describe"},
+                    {"type": "image_url", "image_url": f"data:image/png;base64,{image_data}"},
+                ],
+            }
+        ],
+        model="claude-opus-5-5",
+        llm_provider="anthropic",
+    )
+
+    assert result[0]["content"][1] == {
+        "type": "image",
+        "source": {"type": "base64", "media_type": "image/png", "data": image_data},
+    }
+
+
 EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize(
     "empty_user_message",
     [{"role": "user", "content": None}, {"role": "user", "content": []}, {"role": "user"}],

tests/unit/litellm_core_utils/test_fallback_utils.py

@@ -1,6 +1,7 @@
 """Tests for litellm.litellm_core_utils.fallback_utils."""
 
 import json
+import pathlib
 from typing import Final
 
 import httpx
@@ -8,12 +9,14 @@ import pytest
 import respx
 
 import litellm
+from litellm.litellm_core_utils import litellm_logging
 from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.fallback_utils import (
     async_completion_with_fallbacks,
 )
 from litellm.main import acompletion as python_acompletion
 from litellm.main import completion as python_completion
+from litellm.vector_stores.vector_store_registry import LiteLLM_ManagedVectorStore, VectorStoreRegistry
 
 
 @pytest.mark.asyncio
@@ -444,3 +447,187 @@ def test_completion_fallbacks_sync(
 
     assert response.choices[0].message.content == "sync fallback answer"
     assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("first", "second")
+
+
+CACHE_POINTS: Final = ({"location": "message", "role": "system"},)
+CACHED_MESSAGES: Final = (
+    {"role": "system", "content": "You are a careful assistant."},
+    {"role": "user", "content": "hello"},
+)
+OPENAI_ANSWER: Final = {
+    "id": "chatcmpl-1",
+    "object": "chat.completion",
+    "created": 1,
+    "model": "gpt-6.1-sol",
+    "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
+    "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+}
+ANTHROPIC_ANSWER: Final = {
+    "id": "msg_1",
+    "type": "message",
+    "role": "assistant",
+    "model": "claude-haiku-4-5",
+    "content": [{"type": "text", "text": "ok"}],
+    "stop_reason": "end_turn",
+    "stop_sequence": None,
+    "usage": {"input_tokens": 1, "output_tokens": 1},
+}
+
+
+def _cached_request(**kwargs: object) -> dict[str, object]:
+    return {
+        "messages": [dict(message) for message in CACHED_MESSAGES],
+        "cache_control_injection_points": [dict(point) for point in CACHE_POINTS],
+        **kwargs,
+    }
+
+
+@pytest.mark.asyncio
+async def test_acompletion_fallback_sends_the_fallback_model_the_request_a_direct_call_sends(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    openai_route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
+        return_value=httpx.Response(
+            401,
+            json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}},
+        )
+    )
+    anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
+        return_value=httpx.Response(200, json=ANTHROPIC_ANSWER)
+    )
+
+    await python_acompletion(**_cached_request(model="anthropic/claude-haiku-4-5", api_key="anthropic-key"))
+    await python_acompletion(
+        **_cached_request(
+            model="openai/gpt-6.1-sol",
+            api_key="openai-key",
+            fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}],
+        )
+    )
+
+    assert "prompt_cache_options" in json.loads(openai_route.calls[0].request.content)
+    direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls)
+    assert fallback_body == direct_body
+
+
+@pytest.mark.parametrize("is_async", [True, False])
+@pytest.mark.asyncio
+async def test_cache_dialect_follows_base_url_over_an_api_base_the_call_does_not_use(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, is_async: bool
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=OPENAI_ANSWER)
+    )
+
+    async def send(**kwargs: object) -> None:
+        request: Final = _cached_request(
+            model="openai/gpt-6.1-sol", api_key="openai-key", base_url="https://api.openai.com/v1", **kwargs
+        )
+        if is_async:
+            await python_acompletion(**request)
+        else:
+            python_completion(**request)
+
+    await send()
+    await send(api_base="https://unused-gateway.example.com/v1")
+
+    base_url_only, with_unused_api_base = (json.loads(call.request.content) for call in route.calls)
+    assert with_unused_api_base == base_url_only
+
+
+@pytest.mark.asyncio
+async def test_acompletion_fallback_sends_the_context_a_vector_store_tool_retrieves(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.setattr(
+        litellm,
+        "vector_store_registry",
+        VectorStoreRegistry(
+            vector_stores=[
+                LiteLLM_ManagedVectorStore(
+                    vector_store_id="vs_docs", custom_llm_provider="openai", litellm_params={"api_key": "openai-key"}
+                )
+            ]
+        ),
+    )
+    search_route: Final = respx_mock.post("https://api.openai.com/v1/vector_stores/vs_docs/search").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "object": "vector_store.search_results.page",
+                "search_query": "what is litellm?",
+                "data": [{"score": 1.0, "content": [{"type": "text", "text": "litellm is an LLM gateway"}]}],
+            },
+        )
+    )
+    openai_route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
+        return_value=httpx.Response(
+            401,
+            json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}},
+        )
+    )
+    anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
+        return_value=httpx.Response(200, json=ANTHROPIC_ANSWER)
+    )
+
+    async def send(**kwargs: object) -> None:
+        await python_acompletion(
+            messages=[{"role": "user", "content": "what is litellm?"}],
+            tools=[{"type": "file_search", "vector_store_ids": ["vs_docs"]}],
+            **kwargs,
+        )
+
+    await send(model="anthropic/claude-haiku-4-5", api_key="anthropic-key")
+    await send(
+        model="openai/gpt-6.1-sol",
+        api_key="openai-key",
+        fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}],
+    )
+
+    direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls)
+    assert "litellm is an LLM gateway" in json.dumps(direct_body)
+    assert fallback_body == direct_body
+    assert search_route.call_count <= openai_route.call_count + anthropic_route.call_count
+
+
+@pytest.mark.asyncio
+async def test_acompletion_fallback_sends_the_prompt_the_primary_model_prefix_template_renders(
+    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path
+) -> None:
+    _ = (tmp_path / "brief.prompt").write_text(
+        "---\ninput:\n  schema:\n    question: string\n---\nSystem: Answer in one word.\n\nUser: {{question}}\n"
+    )
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.setattr(litellm, "global_prompt_directory", str(tmp_path))
+    monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [])
+    _ = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
+        return_value=httpx.Response(
+            401,
+            json={"error": {"message": "invalid key", "type": "authentication_error", "param": None, "code": None}},
+        )
+    )
+    anthropic_route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
+        return_value=httpx.Response(200, json=ANTHROPIC_ANSWER)
+    )
+
+    async def send(**kwargs: object) -> None:
+        await python_acompletion(
+            prompt_id="brief",
+            prompt_variables={"question": "What color is the sky?"},
+            messages=[{"role": "user", "content": "placeholder"}],
+            **kwargs,
+        )
+
+    await send(model="dotprompt/anthropic/claude-haiku-4-5", api_key="anthropic-key")
+    await send(
+        model="dotprompt/openai/gpt-6.1-sol",
+        api_key="openai-key",
+        fallbacks=[{"model": "anthropic/claude-haiku-4-5", "api_key": "anthropic-key"}],
+    )
+
+    direct_body, fallback_body = (json.loads(call.request.content) for call in anthropic_route.calls)
+    assert "What color is the sky?" in json.dumps(direct_body)
+    assert fallback_body == direct_body

tests/unit/litellm_core_utils/test_image_handling.py

@@ -288,6 +288,37 @@ def test_convert_url_to_base64_keeps_a_real_content_type(monkeypatch):
     assert result.startswith("data:image/jpeg;base64,")
 
 
+@pytest.mark.parametrize(
+    ("url", "content_type", "expected"),
+    [
+        (
+            "https://docs.example/document.pdf",
+            "application/pdf; qs=0.001",
+            "data:application/pdf;base64,cGRm",
+        ),
+        (
+            "https://img.example/image.png",
+            "image/png; charset=binary",
+            "data:image/png;base64,cGRm",
+        ),
+        (
+            "https://docs.example/document.pdf",
+            "application/octet-stream; charset=binary",
+            "data:application/pdf;base64,cGRm",
+        ),
+    ],
+)
+def test_convert_url_to_base64_uses_bare_content_type(
+    monkeypatch: pytest.MonkeyPatch,
+    url: str,
+    content_type: str,
+    expected: str,
+) -> None:
+    monkeypatch.setattr(litellm, "module_level_client", _stub_image_client(b"pdf", content_type))
+
+    assert convert_url_to_base64(url) == expected
+
+
 def test_convert_url_to_base64_raises_when_no_content_type_is_determinable(monkeypatch):
     monkeypatch.setattr(
         litellm,

tests/unit/litellm_core_utils/test_realtime_streaming.py

@@ -1,7 +1,7 @@
 import asyncio
 import base64
 import json
-from collections.abc import Coroutine, Mapping, Sequence
+from collections.abc import Callable, Coroutine, Mapping, Sequence
 from dataclasses import dataclass
 from typing import Final, Literal
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -14,6 +14,7 @@ from websockets.frames import Close
 import litellm
 from litellm.constants import (
     BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
+    REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
     REALTIME_METERED_USAGE_KEY,
     REALTIME_SESSION_FAILURE_LOGGED_KEY,
     REALTIME_SESSION_SUCCESS_LOGGED_KEY,
@@ -21,6 +22,7 @@ from litellm.constants import (
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.litellm_core_utils.realtime_streaming import (
     RealTimeStreaming,
+    _merge_session_fields,
     client_sent_openai_beta_realtime_header,
 )
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -1255,11 +1257,15 @@ async def test_client_ack_caches_setup_to_prevent_duplicate_session_update_setup
         side_effect=[
             json.dumps({"type": "session.update", "session": {"tools": []}}),
             json.dumps({"type": "session.update", "session": {"tools": []}}),
+            _TEXT_ITEM_CREATE,
+            _RESPONSE_CREATE,
             Exception("client done"),
         ]
     )
+    websocket.send_text = AsyncMock()
 
     provider_config = MagicMock()
+    provider_config.setup_session_update_rejection = MagicMock(return_value=None)
 
     def _transform(message: str, model: str, session_configuration_request=None):
         if session_configuration_request is None:
@@ -2285,6 +2291,7 @@ async def test_guardrail_turn_detection_injected_into_first_session_update_defer
     logging_obj.success_handler = MagicMock()
 
     provider_config = MagicMock()
+    provider_config.builds_setup_from_session_update = MagicMock(return_value=False)
     transformed_messages = []
 
     def mock_transform(msg, model, session_config):
@@ -2351,6 +2358,7 @@ async def test_guardrail_turn_detection_injection_tolerates_non_dict_value(
     logging_obj.success_handler = MagicMock()
 
     provider_config = MagicMock()
+    provider_config.builds_setup_from_session_update = MagicMock(return_value=False)
     transformed_messages = []
 
     def mock_transform(msg, model, session_config):
@@ -2413,6 +2421,7 @@ async def test_subsequent_session_update_cannot_reenable_vad_when_guardrails_act
     logging_obj.success_handler = MagicMock()
 
     provider_config = MagicMock()
+    provider_config.setup_session_update_rejection = MagicMock(return_value=None)
     transformed_messages = []
 
     def mock_transform(msg, model, session_config):
@@ -2465,6 +2474,7 @@ async def test_follow_up_setup_updates_cached_session_configuration_request():
     logging_obj.success_handler = MagicMock()
 
     provider_config = MagicMock()
+    provider_config.setup_session_update_rejection = MagicMock(return_value=None)
     follow_up_setup = json.dumps(
         {
             "setup": {
@@ -2498,6 +2508,636 @@ async def test_follow_up_setup_updates_cached_session_configuration_request():
     assert streaming.session_configuration_request == follow_up_setup
 
 
+def _gemini_default_mode_streaming(
+    monkeypatch: pytest.MonkeyPatch,
+    client_messages: Sequence[str],
+    model: str = "gemini-live-2.5-flash-native-audio",
+    drop_params: bool = False,
+    defer_setup: bool = False,
+) -> tuple[RealTimeStreaming, MagicMock]:
+    monkeypatch.setattr(litellm, "gemini_live_defer_setup", defer_setup, raising=False)
+    from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
+
+    client_ws = MagicMock()
+    client_ws.receive_text = AsyncMock(side_effect=[*client_messages, ConnectionClosed(None, None)])
+    client_ws.send_text = AsyncMock()
+    backend_ws = MagicMock()
+    backend_ws.send = AsyncMock()
+    streaming = RealTimeStreaming(
+        client_ws,
+        backend_ws,
+        MagicMock(),
+        provider_config=GeminiRealtimeConfig(drop_params=drop_params),
+        model=model,
+    )
+    return streaming, backend_ws
+
+
+def _events_sent_to_client(streaming: RealTimeStreaming) -> list[dict]:
+    return [json.loads(call.args[0]) for call in streaming.websocket.send_text.await_args_list]
+
+
+_TEXT_ITEM_CREATE = json.dumps(
+    {
+        "type": "conversation.item.create",
+        "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]},
+    }
+)
+
+_RESPONSE_CREATE = json.dumps({"type": "response.create"})
+
+
+@pytest.mark.asyncio
+async def test_default_mode_sends_default_setup_before_first_non_session_client_message(monkeypatch):
+    """Gemini Live setup is no longer pre-sent at connect, so a client that never
+    sends session.update still gets the provider default setup ahead of its
+    first content frame."""
+    streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [_TEXT_ITEM_CREATE])
+
+    await streaming.client_ack_messages()
+
+    frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"]
+    assert streaming.session_configuration_request == backend_ws.send.await_args_list[0].args[0]
+
+
+@pytest.mark.asyncio
+async def test_default_mode_session_update_first_builds_the_only_setup(monkeypatch):
+    session_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}})
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [session_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash"
+    )
+
+    await streaming.client_ack_messages()
+
+    frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
+
+
+@pytest.mark.asyncio
+async def test_default_mode_merges_every_session_update_sent_before_the_first_content_frame(monkeypatch):
+    """Clients configure a session over several session.update messages. Gemini
+    Live takes one setup, so every update sent before the first content frame is
+    merged into that setup instead of only the first one applying."""
+    instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
+    modalities_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}})
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [instructions_update, modalities_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash"
+    )
+
+    await streaming.client_ack_messages()
+
+    frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
+    assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
+
+
+@pytest.mark.asyncio
+async def test_default_mode_merges_nested_audio_settings_across_session_updates(monkeypatch):
+    vad_update = json.dumps(
+        {"type": "session.update", "session": {"audio": {"input": {"turn_detection": {"silence_duration_ms": 1500}}}}}
+    )
+    voice_update = json.dumps({"type": "session.update", "session": {"audio": {"output": {"voice": "Kore"}}}})
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [vad_update, voice_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash"
+    )
+
+    await streaming.client_ack_messages()
+
+    setup = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"]
+    assert setup["realtimeInputConfig"]["automaticActivityDetection"]["silenceDurationMs"] == 1500
+    assert setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore"
+
+
+def test_merge_session_fields_replaces_nested_mappings_wholesale_past_the_depth_cap():
+    def nested(levels: int, key: str) -> dict:
+        return {key: levels, **({"child": nested(levels - 1, key)} if levels else {})}
+
+    def level(node, depth: int):
+        return node if depth == 0 else level(node["child"], depth - 1)
+
+    merged = _merge_session_fields(nested(10, "keep"), nested(10, "new"))
+
+    assert all({"keep", "new"} <= set(level(merged, depth)) for depth in range(9))
+    assert "keep" not in level(merged, 9)
+    assert level(merged, 9)["new"] == 1
+
+
+@pytest.mark.asyncio
+async def test_default_mode_acknowledges_each_held_session_update_before_any_backend_frame(monkeypatch):
+    """A client that waits for session.updated before sending content must not
+    hang while its updates are held, so each one is acknowledged right away
+    with the modalities the merged setup will carry."""
+    instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
+    modalities_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}})
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [instructions_update, modalities_update], model="gemini-live-2.5-flash"
+    )
+
+    await streaming.client_ack_messages()
+
+    acks = [json.loads(call.args[0]) for call in streaming.websocket.send_text.await_args_list]
+    assert [ack["type"] for ack in acks] == ["session.updated", "session.updated"]
+    assert [ack["session"].get("output_modalities") or ack["session"].get("modalities") for ack in acks] == [
+        ["audio"],
+        ["text"],
+    ]
+    assert acks[1]["session"]["instructions"] == "Be brief."
+    backend_ws.send.assert_not_awaited()
+
+
+_LATE_INSTRUCTIONS_UPDATE = json.dumps(
+    {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Be brief."}}
+)
+
+
+@pytest.mark.asyncio
+async def test_default_mode_answers_a_session_update_after_the_first_content_with_an_error(monkeypatch):
+    streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [_TEXT_ITEM_CREATE, _LATE_INSTRUCTIONS_UPDATE])
+
+    await streaming.client_ack_messages()
+
+    errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"]
+    assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")]
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+
+
+@pytest.mark.asyncio
+async def test_default_mode_drops_a_late_session_update_quietly_under_drop_params(monkeypatch):
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [_TEXT_ITEM_CREATE, _LATE_INSTRUCTIONS_UPDATE], drop_params=True
+    )
+
+    await streaming.client_ack_messages()
+
+    assert _events_sent_to_client(streaming) == []
+    assert backend_ws.send.await_count == 2
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("model", "requested_output"),
+    [
+        ("gemini-live-2.5-flash-native-audio", {"modalities": ["text"]}),
+        ("gemini-live-2.5-flash", {"output_modalities": ["image"]}),
+    ],
+)
+async def test_default_mode_rejects_an_output_modality_the_model_cannot_produce(monkeypatch, model, requested_output):
+    update: Final = json.dumps(
+        {"type": "session.update", "session": {**requested_output, "instructions": "Be brief."}}
+    )
+    streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [update, _TEXT_ITEM_CREATE], model=model)
+
+    await streaming.client_ack_messages()
+
+    events: Final = _events_sent_to_client(streaming)
+    assert [(event["type"], event["error"]["code"]) for event in events] == [("error", "invalid_value")]
+    setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"]
+    assert "systemInstruction" not in setup
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("modalities", "drop_params"),
+    [(["text", "audio"], False), (["text"], True)],
+    ids=["openai-default-pair", "text-dropped-under-drop-params"],
+)
+async def test_default_mode_applies_the_update_with_audio_output_on_an_audio_only_model(
+    monkeypatch, modalities, drop_params
+):
+    update: Final = json.dumps(
+        {"type": "session.update", "session": {"modalities": modalities, "instructions": "Be brief."}}
+    )
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], drop_params=drop_params
+    )
+
+    await streaming.client_ack_messages()
+
+    assert [event["type"] for event in _events_sent_to_client(streaming)] == ["session.updated"]
+    setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"]
+    assert setup["generationConfig"]["responseModalities"] == ["AUDIO"]
+    assert setup["systemInstruction"]["parts"][0]["text"] == "Be brief."
+
+
+@pytest.mark.asyncio
+async def test_default_mode_sends_no_extra_session_updated_when_gemini_confirms_the_setup(monkeypatch):
+    """Each held session.update was already acknowledged, so Gemini's setupComplete
+    must not reach the client as one more session.updated nobody asked for."""
+    monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False)
+    from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
+
+    model: Final = "gemini-live-2.5-flash-native-audio"
+    config: Final = GeminiRealtimeConfig()
+    client_ws: Final = MagicMock()
+    client_ws.send_text = AsyncMock()
+    backend_ws: Final = MagicMock()
+    backend_ws.send = AsyncMock()
+    backend_ws.recv = AsyncMock(side_effect=[json.dumps({"setupComplete": {}}).encode(), ConnectionClosed(None, None)])
+    logging_obj: Final = _mock_logging()
+    logging_obj.litellm_trace_id = "trace_lazy"
+    logging_obj.async_success_handler = AsyncMock()
+    logging_obj.success_handler = MagicMock()
+    streaming: Final = RealTimeStreaming(client_ws, backend_ws, logging_obj, provider_config=config, model=model)
+    streaming.session_created_sent_to_client = True
+    streaming.session_configuration_request = config.session_configuration_request(model)
+
+    await streaming.backend_to_client_send_messages()
+
+    assert [json.loads(call.args[0])["type"] for call in client_ws.send_text.await_args_list] == []
+
+
+@pytest.mark.asyncio
+async def test_default_mode_folds_transcription_guardrail_vad_disable_into_the_merged_setup(monkeypatch):
+    """A transcription guardrail gates every turn, so its auto-response disable
+    has to land in the one setup built from the held session.updates, injected
+    once rather than into every update that gets merged."""
+    monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()])
+    debug = MagicMock()
+    monkeypatch.setattr("litellm.litellm_core_utils.realtime_streaming.verbose_logger.debug", debug)
+    instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
+    vad_update = json.dumps(
+        {"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "silence_duration_ms": 900}}}
+    )
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [instructions_update, vad_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash"
+    )
+
+    await streaming.client_ack_messages()
+
+    frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    assert frames[0]["setup"]["realtimeInputConfig"]["automaticActivityDetection"] == {"disabled": True}
+    assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
+    assert streaming._guardrail_turn_detection_update_sent is True
+    injected = [call for call in debug.call_args_list if str(call.args[0]).startswith("Injected turn_detection")]
+    assert len(injected) == 1
+
+
+def _blocking_text_guardrail() -> CustomGuardrail:
+    class _BlockingTextGuardrail(CustomGuardrail):
+        async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+            if any("zebrafruit" in text for text in inputs.get("texts", [])):
+                raise ValueError("Content blocked")
+            return inputs
+
+    return _BlockingTextGuardrail(
+        guardrail_name="test_text_guard", event_hook=GuardrailEventHooks.pre_call, default_on=True
+    )
+
+
+@pytest.mark.asyncio
+async def test_default_mode_sends_the_setup_before_a_guardrail_blocks_the_first_message(monkeypatch):
+    """Gemini Live closes the socket when content arrives before the setup, so a
+    guardrail that blocks the first typed message must not reach the backend first."""
+    monkeypatch.setattr(litellm, "callbacks", [_blocking_text_guardrail()])
+    instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
+    blocked_item = json.dumps(
+        {
+            "type": "conversation.item.create",
+            "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "zebrafruit"}]},
+        }
+    )
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [instructions_update, blocked_item], model="gemini-live-2.5-flash"
+    )
+
+    await streaming.client_ack_messages()
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
+    assert "zebrafruit" not in json.dumps(frames[1])
+    errors: Final = [event["error"]["type"] for event in _events_sent_to_client(streaming) if event["type"] == "error"]
+    assert errors == ["guardrail_violation"]
+
+
+class _SteppedClock:
+    def __init__(self, epoch: float) -> None:
+        self.now = epoch
+
+    def __call__(self) -> float:
+        return self.now
+
+
+def _client_ws_receiving(*steps: Callable[[], Coroutine[object, object, str]]) -> MagicMock:
+    remaining: Final = iter(steps)
+
+    async def receive_text() -> str:
+        return await next(remaining)()
+
+    client_ws: Final = MagicMock()
+    client_ws.receive_text = receive_text
+    client_ws.send_text = AsyncMock()
+    return client_ws
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("epoch", [0.0, 1e9])
+async def test_default_mode_sends_the_held_setup_when_no_content_arrives_before_the_deadline(monkeypatch, epoch):
+    """Vertex AI refuses a setup sent minutes after connecting, so an idle session
+    sends what it holds at the deadline and answers any later session.update as late."""
+    monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False)
+    from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
+
+    clock: Final = _SteppedClock(epoch)
+    setup_sent: Final = asyncio.Event()
+
+    async def instructions_update_then_the_deadline_passes() -> str:
+        clock.now += REALTIME_LAZY_SETUP_DEADLINE_SECONDS
+        return json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
+
+    async def late_update_once_the_setup_is_out() -> str:
+        await setup_sent.wait()
+        return json.dumps(
+            {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}}
+        )
+
+    async def disconnect() -> str:
+        raise ConnectionClosed(None, None)
+
+    backend_ws: Final = MagicMock()
+    backend_ws.send = AsyncMock(side_effect=lambda frame: setup_sent.set())
+    streaming: Final = RealTimeStreaming(
+        _client_ws_receiving(instructions_update_then_the_deadline_passes, late_update_once_the_setup_is_out, disconnect),
+        backend_ws,
+        MagicMock(),
+        provider_config=GeminiRealtimeConfig(),
+        model="gemini-live-2.5-flash",
+        clock=clock,
+    )
+
+    await asyncio.wait_for(streaming.client_ack_messages(), timeout=5)
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup"]
+    assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
+    events: Final = _events_sent_to_client(streaming)
+    assert [(event["type"], event.get("error", {}).get("code")) for event in events] == [
+        ("session.updated", None),
+        ("error", "session_update_not_applied"),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_default_mode_refuses_held_session_updates_past_the_size_cap(monkeypatch):
+    def instructions_update(text: str) -> str:
+        return json.dumps({"type": "session.update", "event_id": f"evt_{text[0]}", "session": {"instructions": text}})
+
+    half_the_cap: Final = RealTimeStreaming._MAX_BUFFERED_BYTES // 2
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch,
+        [instructions_update("a" * half_the_cap), instructions_update("b" * half_the_cap), _TEXT_ITEM_CREATE, _RESPONSE_CREATE],
+        model="gemini-live-2.5-flash",
+    )
+
+    await streaming.client_ack_messages()
+
+    events: Final = _events_sent_to_client(streaming)
+    assert [(event["type"], event.get("error", {}).get("event_id")) for event in events] == [
+        ("session.updated", None),
+        ("error", "evt_b"),
+    ]
+    assert events[1]["error"]["code"] == "session_update_too_large"
+    setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"]
+    assert setup["systemInstruction"]["parts"][0]["text"].startswith("a")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("defer_setup", [False, True])
+async def test_a_session_update_between_the_first_items_and_response_create_joins_the_setup(monkeypatch, defer_setup):
+    modalities_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}})
+    instructions_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch,
+        [modalities_update, _TEXT_ITEM_CREATE, instructions_update, _RESPONSE_CREATE],
+        model="gemini-live-2.5-flash",
+        defer_setup=defer_setup,
+    )
+
+    await streaming.client_ack_messages()
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
+    assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
+    assert [event["type"] for event in _events_sent_to_client(streaming)] == ["session.updated", "session.updated"]
+
+
+@pytest.mark.asyncio
+async def test_default_mode_applies_a_session_update_that_arrives_while_audio_streams_in(monkeypatch):
+    modalities_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}})
+    append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})
+    instructions_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
+    commit: Final = json.dumps({"type": "input_audio_buffer.commit"})
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [modalities_update, append, instructions_update, commit], model="gemini-live-2.5-flash"
+    )
+
+    await streaming.client_ack_messages()
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames][:2] == ["setup", "realtimeInput"]
+    assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
+    assert frames[1]["realtimeInput"]["audio"]["data"] == "AAAA"
+    assert "error" not in [event["type"] for event in _events_sent_to_client(streaming)]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("epoch", [0.0, 1e9])
+async def test_held_items_go_out_with_the_setup_when_no_response_create_follows(monkeypatch, epoch):
+    monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False)
+    from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
+
+    item_sent: Final = asyncio.Event()
+
+    async def instructions_update() -> str:
+        return json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
+
+    async def text_item() -> str:
+        return _TEXT_ITEM_CREATE
+
+    async def disconnect_once_the_item_is_out() -> str:
+        await item_sent.wait()
+        raise ConnectionClosed(None, None)
+
+    backend_ws: Final = MagicMock()
+    backend_ws.send = AsyncMock(
+        side_effect=lambda frame: item_sent.set() if "clientContent" in json.loads(frame) else None
+    )
+    streaming: Final = RealTimeStreaming(
+        _client_ws_receiving(instructions_update, text_item, disconnect_once_the_item_is_out),
+        backend_ws,
+        MagicMock(),
+        provider_config=GeminiRealtimeConfig(),
+        model="gemini-live-2.5-flash",
+        lazy_setup_content_grace_seconds=0,
+        clock=_SteppedClock(epoch),
+    )
+
+    await asyncio.wait_for(streaming.client_ack_messages(), timeout=5)
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
+
+
+@pytest.mark.asyncio
+async def test_default_mode_acknowledges_an_unchanged_session_update_after_content(monkeypatch):
+    """Pipecat resends its unchanged session.update right after its first
+    conversation items and treats any error event as fatal."""
+    session_update: Final = json.dumps(
+        {"type": "session.update", "session": {"modalities": ["text"], "instructions": "Be brief."}}
+    )
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [session_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE, session_update], model="gemini-live-2.5-flash"
+    )
+
+    await streaming.client_ack_messages()
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    events: Final = _events_sent_to_client(streaming)
+    assert [event["type"] for event in events] == ["session.updated", "session.updated"]
+    assert events[0]["session"] == events[1]["session"]
+
+
+@pytest.mark.asyncio
+async def test_deferred_setup_merges_session_updates_until_the_first_content(monkeypatch):
+    """Deferred setup merges every session.update sent before the first content,
+    the same as the default mode, and answers a changed one after it as late."""
+    monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False)
+    from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
+
+    client_ws: Final = MagicMock()
+    client_ws.receive_text = AsyncMock(
+        side_effect=[
+            json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}),
+            json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}),
+            _TEXT_ITEM_CREATE,
+            _RESPONSE_CREATE,
+            json.dumps(
+                {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}}
+            ),
+            ConnectionClosed(None, None),
+        ]
+    )
+    client_ws.send_text = AsyncMock()
+    backend_ws: Final = MagicMock()
+    backend_ws.send = AsyncMock()
+    streaming: Final = RealTimeStreaming(
+        client_ws, backend_ws, MagicMock(), provider_config=GeminiRealtimeConfig(), model="gemini-live-2.5-flash"
+    )
+
+    await streaming.client_ack_messages()
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
+    assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
+    errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"]
+    assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")]
+
+
+@pytest.mark.asyncio
+async def test_deferred_setup_sends_the_default_setup_before_text_content_that_comes_first(monkeypatch):
+    """Gemini Live closes the socket on content sent before any setup, so text that arrives
+    before a session.update in deferred mode gets the model's default setup first."""
+    monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False)
+    from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
+
+    client_ws: Final = MagicMock()
+    client_ws.receive_text = AsyncMock(
+        side_effect=[
+            _TEXT_ITEM_CREATE,
+            json.dumps({"type": "session.update", "event_id": "evt_late", "session": {"modalities": ["text"]}}),
+            ConnectionClosed(None, None),
+        ]
+    )
+    client_ws.send_text = AsyncMock()
+    backend_ws: Final = MagicMock()
+    backend_ws.send = AsyncMock()
+    streaming: Final = RealTimeStreaming(
+        client_ws, backend_ws, MagicMock(), provider_config=GeminiRealtimeConfig(), model="gemini-live-2.5-flash"
+    )
+
+    await streaming.client_ack_messages()
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"]
+    assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")]
+
+
+@pytest.mark.asyncio
+async def test_deferred_setup_sends_a_session_update_that_follows_buffered_audio_at_once(monkeypatch):
+    append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})
+    commit: Final = json.dumps({"type": "input_audio_buffer.commit"})
+    session_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be a pirate."}})
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch, [append, commit, session_update], model="gemini-live-2.5-flash", defer_setup=True
+    )
+
+    await streaming.client_ack_messages()
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup"]
+    assert "Be a pirate." in json.dumps(frames[0]["setup"]["systemInstruction"])
+
+
+@pytest.mark.parametrize("defer_setup", [False, True])
+@pytest.mark.asyncio
+async def test_a_transcription_guardrail_refuses_gemini_audio_it_cannot_screen(monkeypatch, defer_setup):
+    monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()])
+    session_update: Final = json.dumps(
+        {"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "create_response": False}}}
+    )
+    append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})
+    commit: Final = json.dumps({"type": "input_audio_buffer.commit", "event_id": "commit_1"})
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch,
+        [session_update, append, append, commit, json.dumps({"type": "response.create"}), _TEXT_ITEM_CREATE],
+        model="gemini-live-2.5-flash",
+        defer_setup=defer_setup,
+    )
+
+    await streaming.client_ack_messages()
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    refusals: Final = [
+        event["error"]["event_id"]
+        for event in _events_sent_to_client(streaming)
+        if event["type"] == "error" and event["error"]["code"] == "audio_input_not_screenable"
+    ]
+    assert refusals == [None, "commit_1"]
+
+
+@pytest.mark.parametrize("defer_setup", [False, True])
+@pytest.mark.parametrize("control_type", ["response.cancel", "conversation.item.delete", "input_audio_buffer.clear"])
+@pytest.mark.asyncio
+async def test_a_control_event_before_session_update_leaves_the_setup_open(monkeypatch, defer_setup, control_type):
+    control_event: Final = json.dumps({"type": control_type, "item_id": "item_x"})
+    session_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}})
+    streaming, backend_ws = _gemini_default_mode_streaming(
+        monkeypatch,
+        [control_event, session_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE],
+        model="gemini-live-2.5-flash",
+        defer_setup=defer_setup,
+    )
+
+    await streaming.client_ack_messages()
+
+    frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
+    assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
+    assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
+    assert [event["type"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] == []
+
+
 @pytest.mark.asyncio
 async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeypatch):
     """Pipecat may send audio before session.update when setup is deferred."""
@@ -2533,14 +3173,18 @@ async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeyp
 
 
 @pytest.mark.asyncio
-async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeypatch):
+async def test_deferred_setup_sends_the_held_session_update_when_more_audio_arrives(monkeypatch):
     monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False)
     from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
 
     client_ws = MagicMock()
+    client_ws.send_text = AsyncMock()
     audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AA=="})
     session_update = json.dumps({"type": "session.update", "session": {"modalities": ["audio"]}})
-    client_ws.receive_text = AsyncMock(side_effect=[audio_msg, session_update, ConnectionClosed(None, None)])
+    later_audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AQ=="})
+    client_ws.receive_text = AsyncMock(
+        side_effect=[audio_msg, session_update, later_audio_msg, ConnectionClosed(None, None)]
+    )
     backend_ws = MagicMock()
     backend_ws.send = AsyncMock()
     logging_obj: Final = _mock_logging()
@@ -2560,7 +3204,7 @@ async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeyp
     sent_payload = json.loads(backend_ws.send.await_args_list[0].args[0])
     assert "setup" in sent_payload
     assert "realtimeInput" not in sent_payload
-    assert streaming._pending_messages_until_setup == [audio_msg]
+    assert streaming._pending_messages_until_setup == [audio_msg, later_audio_msg]
 
 
 @pytest.mark.asyncio
@@ -3523,6 +4167,24 @@ async def test_upstream_refusal_after_a_synthetic_session_created_still_logs_a_f
     assert session.logging.logged_sessions == ()
 
 
+class _LazySetupRealtimeConfig(_PassthroughRealtimeConfig):
+    def builds_setup_from_session_update(self) -> bool:
+        return True
+
+
+@pytest.mark.asyncio
+async def test_upstream_outage_before_a_held_setup_was_sent_logs_the_session_not_a_refusal():
+    upstream_close: Final = ConnectionClosed(Close(1011, "upstream outage"), None)
+    session: Final = _relay_session(
+        _client_ws_that_never_sends(), _backend_ws_closing_with(upstream_close), _LazySetupRealtimeConfig()
+    )
+
+    await session.run()
+
+    assert session.logging.logged_failures == ()
+    assert len(session.logging.logged_sessions) == 1
+
+
 @pytest.mark.asyncio
 async def test_upstream_close_after_relayed_events_still_logs_the_session_as_success():
     client_ws: Final = _client_ws_that_never_sends()

tests/unit/litellm_core_utils/test_streaming_handler.py

@@ -2,12 +2,15 @@ import asyncio
 import datetime
 import itertools
 import json
+import os
 import queue
+import subprocess
 import sys
 import threading
 import time
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
+from pathlib import Path
 from types import CodeType, FrameType, MappingProxyType
 from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, cast
 from unittest.mock import AsyncMock, MagicMock, Mock, patch
@@ -33,6 +36,7 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler
 from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
 from litellm.responses.mcp.chat_completions_handler import _add_mcp_metadata_to_response
 from litellm.types.utils import (
+    ChoiceLogprobs,
     CompletionTokensDetailsWrapper,
     Delta,
     ModelResponse,
@@ -1624,6 +1628,356 @@ async def test_stream_with_null_choices_chunk_finishes_normally(
     assert [choice.finish_reason for choice in streamed_choices if choice.finish_reason] == ["stop"]
 
 
+def _token_logprobs(token: str) -> dict[str, object]:
+    return {
+        "content": [{"token": token, "logprob": -0.5, "bytes": list(token.encode()), "top_logprobs": []}],
+        "refusal": None,
+    }
+
+
+@pytest.mark.parametrize("async_stream", [False, True])
+@pytest.mark.parametrize("transport", ["sdk", "http_handler"])
+@pytest.mark.parametrize(
+    ("finish_chunk_extra", "stream_options", "expected_usage_totals"),
+    [
+        pytest.param({}, None, [], id="finish-chunk-with-logprobs"),
+        pytest.param(
+            {"usage": {"prompt_tokens": 3, "completion_tokens": 3, "total_tokens": 6}},
+            {"include_usage": True},
+            [6],
+            id="finish-chunk-with-logprobs-and-usage",
+        ),
+    ],
+)
+async def test_stream_finish_chunk_with_logprobs_reaches_caller_as_litellm_logprobs(
+    async_stream: bool,
+    transport: str,
+    finish_chunk_extra: dict[str, object],
+    stream_options: dict[str, bool] | None,
+    expected_usage_totals: list[int],
+    logging_obj: Logging,
+) -> None:
+    upstream_tokens: Final = ("Hel", "lo", "<|return|>")
+    upstream_chunks: Final = (
+        {
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": {"role": "assistant", "content": "Hel"},
+                    "logprobs": _token_logprobs("Hel"),
+                    "finish_reason": None,
+                }
+            ]
+        },
+        {
+            "choices": [
+                {"index": 0, "delta": {"content": "lo"}, "logprobs": _token_logprobs("lo"), "finish_reason": None}
+            ]
+        },
+        {
+            "choices": [{"index": 0, "delta": {}, "logprobs": _token_logprobs("<|return|>"), "finish_reason": "stop"}],
+            **finish_chunk_extra,
+        },
+    )
+    body: Final = (
+        "".join(
+            "data: "
+            + json.dumps(
+                {
+                    "id": "chatcmpl-logprobs",
+                    "object": "chat.completion.chunk",
+                    "created": 1,
+                    "model": "test-model",
+                    **chunk,
+                }
+            )
+            + "\n\n"
+            for chunk in upstream_chunks
+        )
+        + "data: [DONE]\n\n"
+    )
+    raw_response: Final = httpx.Response(
+        200,
+        content=body,
+        headers={"content-type": "text/event-stream"},
+        request=httpx.Request("POST", "https://provider.example/v1/chat/completions"),
+    )
+    client: Final = AsyncOpenAI(api_key="unused") if async_stream else OpenAI(api_key="unused")
+    stream: Final = (
+        (
+            AsyncStream(cast_to=ChatCompletionChunk, response=raw_response, client=client)
+            if isinstance(client, AsyncOpenAI)
+            else Stream(cast_to=ChatCompletionChunk, response=raw_response, client=client)
+        )
+        if transport == "sdk"
+        else OpenAIChatCompletionStreamingHandler(
+            streaming_response=raw_response.aiter_lines() if async_stream else raw_response.iter_lines(),
+            sync_stream=not async_stream,
+        )
+    )
+    response: Final = CustomStreamWrapper(
+        completion_stream=stream,
+        model="test-model",
+        logging_obj=logging_obj,
+        custom_llm_provider="openai" if transport == "sdk" else "hosted_vllm",
+        stream_options=stream_options,
+    )
+
+    try:
+        chunks: Final = [chunk async for chunk in response] if async_stream else list(response)
+    finally:
+        if isinstance(client, AsyncOpenAI):
+            await client.close()
+        else:
+            client.close()
+
+    wire_choices: Final = tuple(
+        itertools.chain.from_iterable(
+            json.loads(chunk.model_dump_json(exclude_none=True, exclude_unset=True))["choices"] for chunk in chunks
+        )
+    )
+    finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")]
+    streamed_logprobs: Final = tuple(
+        choice.logprobs
+        for choice in itertools.chain.from_iterable(chunk.choices for chunk in chunks)
+        if choice.logprobs is not None
+    )
+    streamed_tokens: Final = tuple(
+        entry.token for entry in itertools.chain.from_iterable(logprobs.content or () for logprobs in streamed_logprobs)
+    )
+
+    assert [choice["finish_reason"] for choice in finish_choices] == ["stop"]
+    assert [entry["token"] for entry in finish_choices[0]["logprobs"]["content"]] == [upstream_tokens[-1]]
+    assert all(isinstance(logprobs, ChoiceLogprobs) for logprobs in streamed_logprobs)
+    assert streamed_tokens == upstream_tokens
+    assert [chunk.usage.total_tokens for chunk in chunks if getattr(chunk, "usage", None)] == expected_usage_totals
+
+
+async def _sdk_stream_chunks(
+    upstream_chunks: Sequence[Mapping[str, object]],
+    async_stream: bool,
+    logging_obj: Logging,
+    stream_options: dict[str, bool] | None = None,
+) -> list[ModelResponseStream]:
+    body: Final = (
+        "".join(
+            "data: "
+            + json.dumps(
+                {
+                    "id": "chatcmpl-logprobs",
+                    "object": "chat.completion.chunk",
+                    "created": 1,
+                    "model": "test-model",
+                    **chunk,
+                }
+            )
+            + "\n\n"
+            for chunk in upstream_chunks
+        )
+        + "data: [DONE]\n\n"
+    )
+    raw_response: Final = httpx.Response(
+        200,
+        content=body,
+        headers={"content-type": "text/event-stream"},
+        request=httpx.Request("POST", "https://provider.example/v1/chat/completions"),
+    )
+    client: Final = AsyncOpenAI(api_key="unused") if async_stream else OpenAI(api_key="unused")
+    stream: Final = (
+        AsyncStream(cast_to=ChatCompletionChunk, response=raw_response, client=client)
+        if isinstance(client, AsyncOpenAI)
+        else Stream(cast_to=ChatCompletionChunk, response=raw_response, client=client)
+    )
+    response: Final = CustomStreamWrapper(
+        completion_stream=stream,
+        model="test-model",
+        logging_obj=logging_obj,
+        custom_llm_provider="openai",
+        stream_options=stream_options,
+    )
+    try:
+        return [chunk async for chunk in response] if async_stream else list(response)
+    finally:
+        if isinstance(client, AsyncOpenAI):
+            await client.close()
+        else:
+            client.close()
+
+
+@pytest.mark.parametrize("async_stream", [False, True])
+@pytest.mark.parametrize(
+    ("upstream_logprobs", "finish_chunk_extra", "stream_options", "expected_usage_totals"),
+    [
+        pytest.param(
+            {"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None},
+            {},
+            None,
+            [],
+            id="null-logprob",
+        ),
+        pytest.param(
+            {"content": [{"token": "<|return|>", "logprob": None}]},
+            {},
+            None,
+            [],
+            id="null-logprob-keeps-only-the-keys-upstream-sent",
+        ),
+        pytest.param(-0.25, {}, None, [], id="scalar-logprobs"),
+        pytest.param(
+            {"content": [{"token": "<|return|>", "bytes": None, "logprob": None, "top_logprobs": None}], "refusal": None},
+            {"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}},
+            {"include_usage": True},
+            [5],
+            id="null-logprob-and-usage",
+        ),
+    ],
+)
+async def test_stream_with_nonstandard_logprobs_passes_them_through_and_finishes(
+    async_stream: bool,
+    upstream_logprobs: object,
+    finish_chunk_extra: dict[str, object],
+    stream_options: dict[str, bool] | None,
+    expected_usage_totals: list[int],
+    logging_obj: Logging,
+) -> None:
+    chunks: Final = await _sdk_stream_chunks(
+        (
+            {
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta": {"role": "assistant", "content": "hi"},
+                        "logprobs": upstream_logprobs,
+                        "finish_reason": None,
+                    }
+                ]
+            },
+            {
+                "choices": [{"index": 0, "delta": {}, "logprobs": upstream_logprobs, "finish_reason": "stop"}],
+                **finish_chunk_extra,
+            },
+        ),
+        async_stream,
+        logging_obj,
+        stream_options,
+    )
+
+    wire_choices: Final = tuple(
+        itertools.chain.from_iterable(
+            json.loads(chunk.model_dump_json(exclude_none=True, exclude_unset=True))["choices"] for chunk in chunks
+        )
+    )
+    logprobs_choices: Final = [choice for choice in wire_choices if "logprobs" in choice]
+
+    assert [choice.get("finish_reason") for choice in logprobs_choices] == [None, "stop"]
+    assert [choice["logprobs"] for choice in logprobs_choices] == [upstream_logprobs, upstream_logprobs]
+    assert [chunk.usage.total_tokens for chunk in chunks if getattr(chunk, "usage", None)] == expected_usage_totals
+
+
+@pytest.mark.parametrize("async_stream", [False, True])
+@pytest.mark.parametrize("logprobs_field", ["content", "refusal"])
+async def test_stream_logprobs_without_top_logprobs_reach_caller_as_litellm_logprobs(
+    async_stream: bool, logprobs_field: Literal["content", "refusal"], logging_obj: Logging
+) -> None:
+    chunks: Final = await _sdk_stream_chunks(
+        tuple(
+            {
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta": delta,
+                        "logprobs": {logprobs_field: [{"token": token, "logprob": -0.5}]},
+                        "finish_reason": finish_reason,
+                    }
+                ]
+            }
+            for token, delta, finish_reason in (
+                ("hi", {"role": "assistant", "content": "hi"}, None),
+                ("<|return|>", {}, "stop"),
+            )
+        ),
+        async_stream,
+        logging_obj,
+    )
+
+    streamed_logprobs: Final = tuple(
+        choice.logprobs
+        for choice in itertools.chain.from_iterable(chunk.choices for chunk in chunks)
+        if choice.logprobs is not None
+    )
+    streamed_entries: Final = tuple(
+        itertools.chain.from_iterable(
+            (logprobs.content if logprobs_field == "content" else logprobs.refusal) or ()
+            for logprobs in streamed_logprobs
+        )
+    )
+
+    assert [(entry.token, entry.logprob, entry.top_logprobs) for entry in streamed_entries] == [
+        ("hi", -0.5, []),
+        ("<|return|>", -0.5, []),
+    ]
+
+
+FINISH_ONLY_LOGPROBS_STREAM_SCRIPT: Final = """
+import json
+import sys
+
+sys.path.insert(0, sys.argv[1])
+
+import httpx
+from openai import OpenAI
+
+import litellm
+
+upstream_choices = (
+    {"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": None},
+    {
+        "index": 0,
+        "delta": {},
+        "logprobs": {"content": [{"token": "<|return|>", "logprob": -0.5, "top_logprobs": []}]},
+        "finish_reason": "stop",
+    },
+)
+body = "".join(
+    "data: "
+    + json.dumps({"id": "c", "object": "chat.completion.chunk", "created": 1, "model": "m", "choices": [choice]})
+    + "\\n\\n"
+    for choice in upstream_choices
+) + "data: [DONE]\\n\\n"
+client = OpenAI(
+    api_key="unused",
+    base_url="https://provider.example/v1",
+    http_client=httpx.Client(
+        transport=httpx.MockTransport(
+            lambda request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"})
+        )
+    ),
+)
+stream = litellm.completion(
+    model="openai/m", messages=[{"role": "user", "content": "hi"}], stream=True, logprobs=True, client=client
+)
+print(json.dumps([json.loads(chunk.model_dump_json(exclude_none=True)) for chunk in stream]))
+"""
+
+
+def test_stream_with_logprobs_only_on_finish_chunk_serializes_in_a_fresh_process() -> None:
+    result: Final = subprocess.run(
+        [sys.executable, "-I", "-c", FINISH_ONLY_LOGPROBS_STREAM_SCRIPT, str(Path(litellm.__file__).parent.parent)],
+        capture_output=True,
+        text=True,
+        timeout=120,
+        env={**os.environ, "LITELLM_LOCAL_MODEL_COST_MAP": "True"},
+    )
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    wire_choices: Final = tuple(
+        itertools.chain.from_iterable(chunk["choices"] for chunk in json.loads(result.stdout.strip().splitlines()[-1]))
+    )
+    finish_choices: Final = [choice for choice in wire_choices if choice.get("finish_reason")]
+    assert [choice["finish_reason"] for choice in finish_choices] == ["stop"]
+    assert [entry["token"] for entry in finish_choices[0]["logprobs"]["content"]] == ["<|return|>"]
+
+
 def _hosted_vllm_stream_wrapper(logging_obj: Logging, error_payload: dict) -> CustomStreamWrapper:
     """A CustomStreamWrapper over the real OpenAI-compatible line iterator,
     fed an HTTP 200 SSE body that carries an in-body error payload the way

tests/unit/llms/custom_httpx/test_llm_http_handler.py

@@ -14,6 +14,7 @@ import httpcore
 import httpx
 import pytest
 import respx
+import websockets.exceptions
 from botocore.credentials import RefreshableCredentials
 from h2.events import ConnectionTerminated
 
@@ -2787,6 +2788,194 @@ async def test_async_realtime_error_event_send_failure_still_closes():
     assert client_ws.events == [("close", (1011, "Internal server error: vertex token refresh exploded"))]
 
 
+_GEMINI_LIVE_SETUP_COMPLETE = json.dumps({"setupComplete": {}})
+_GEMINI_LIVE_TEXT_TURN = json.dumps({"serverContent": {"modelTurn": {"parts": [{"text": "Blue."}]}}})
+_GEMINI_LIVE_TURN_COMPLETE = json.dumps({"serverContent": {"turnComplete": True}})
+
+
+class _ScriptedRealtimeBackend:
+    """Backend socket that answers the first setup frame with the scripted replies
+    and closes once the client is done, so both bridge loops run to completion."""
+
+    def __init__(self, replies_after_setup):
+        self.sent = []
+        self.closed = asyncio.Event()
+        self._replies_after_setup = list(replies_after_setup)
+        self._inbox = asyncio.Queue()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        return None
+
+    async def send(self, payload):
+        self.sent.append(payload)
+        if "setup" in json.loads(payload):
+            for reply in self._replies_after_setup:
+                self._inbox.put_nowait(reply)
+
+    def close_after_replies(self):
+        self._inbox.put_nowait(None)
+
+    async def recv(self, decode=True):
+        reply = await self._inbox.get()
+        if reply is None:
+            self.closed.set()
+            raise websockets.exceptions.ConnectionClosedOK(None, None)
+        return reply
+
+
+class _ScriptedRealtimeClient:
+    def __init__(self, messages, backend):
+        self.scope = {"headers": []}
+        self.exceptions = websockets.exceptions
+        self.sent = []
+        self._messages = list(messages)
+        self._backend = backend
+
+    async def send_text(self, payload):
+        self.sent.append(payload)
+
+    async def receive_text(self):
+        if self._messages:
+            return self._messages.pop(0)
+        self._backend.close_after_replies()
+        await self._backend.closed.wait()
+        raise websockets.exceptions.ConnectionClosedOK(None, None)
+
+    async def close(self, code=None, reason=None):
+        return None
+
+
+def _realtime_logging_obj():
+    logging_obj = Mock()
+    logging_obj.litellm_trace_id = "trace-lit-6974"
+    logging_obj.async_success_handler = AsyncMock()
+    logging_obj.dispatch_failure_handlers = AsyncMock()
+    return logging_obj
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "model, session, backend_modality, client_modality, backend_turn, drop_params",
+    [
+        (
+            "gemini-live-2.5-flash",
+            {"type": "realtime", "output_modalities": ["text"]},
+            "TEXT",
+            "text",
+            [_GEMINI_LIVE_TEXT_TURN],
+            False,
+        ),
+        ("gemini-live-2.5-flash", {"modalities": ["text"]}, "TEXT", "text", [_GEMINI_LIVE_TEXT_TURN], False),
+        (
+            "gemini-live-2.5-flash-native-audio",
+            {"type": "realtime", "output_modalities": ["text"]},
+            "AUDIO",
+            "audio",
+            [],
+            True,
+        ),
+    ],
+)
+async def test_async_realtime_vertex_first_session_update_builds_the_only_setup(
+    model, session, backend_modality, client_modality, backend_turn, drop_params: bool, monkeypatch
+):
+    """Regression for text-only Vertex Live sessions answered in audio: the
+    connect-time default setup (AUDIO) used to go out before the client's
+    session.update, and Gemini accepts only one setup per connection, so the
+    requested modality was dropped and billed at the audio rate. The setup now
+    goes out on the client's first message, built from session.update when that
+    comes first. Under drop_params an audio-only model still gets AUDIO, and the
+    client learns the downgrade from session.updated."""
+    from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig
+
+    monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False)
+    backend = _ScriptedRealtimeBackend([_GEMINI_LIVE_SETUP_COMPLETE, *backend_turn, _GEMINI_LIVE_TURN_COMPLETE])
+    client_ws = _ScriptedRealtimeClient(
+        [
+            json.dumps({"type": "session.update", "session": session}),
+            json.dumps(
+                {
+                    "type": "conversation.item.create",
+                    "item": {
+                        "type": "message",
+                        "role": "user",
+                        "content": [{"type": "input_text", "text": "What color is the sky? One word."}],
+                    },
+                }
+            ),
+            json.dumps({"type": "response.create"}),
+        ],
+        backend,
+    )
+    handler = BaseLLMHTTPHandler()
+
+    with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=backend)):
+        await asyncio.wait_for(
+            handler.async_realtime(
+                model=model,
+                websocket=client_ws,
+                logging_obj=_realtime_logging_obj(),
+                provider_config=VertexAIRealtimeConfig(
+                    access_token="fake-token", project="fake-project", location="us-central1", drop_params=drop_params
+                ),
+                headers={},
+            ),
+            timeout=10,
+        )
+
+    backend_frames = [json.loads(frame) for frame in backend.sent]
+    setups = [frame["setup"] for frame in backend_frames if "setup" in frame]
+    assert len(setups) == 1
+    assert "setup" in backend_frames[0]
+    assert setups[0]["generationConfig"]["responseModalities"] == [backend_modality]
+    assert any("clientContent" in frame for frame in backend_frames[1:])
+
+    client_events = [json.loads(frame) for frame in client_ws.sent]
+    event_types = [event["type"] for event in client_events]
+    assert event_types[:2] == ["session.created", "session.updated"]
+    assert client_events[1]["session"]["modalities"] == [client_modality]
+    if backend_turn:
+        assert "response.output_text.delta" in event_types
+        assert "response.output_audio_transcript.delta" not in event_types
+
+
+@pytest.mark.asyncio
+async def test_async_realtime_sends_setup_at_connect_when_provider_cannot_build_it_from_session_update():
+    """Bedrock Nova Sonic style providers keep the connect-time setup and get no
+    synthetic session.created."""
+    setup = json.dumps({"event": {"sessionStart": {}}})
+    backend = _ScriptedRealtimeBackend([])
+    client_ws = _ScriptedRealtimeClient([], backend)
+    provider_config = Mock()
+    provider_config.get_complete_url.return_value = "wss://backend.example/live"
+    provider_config.validate_environment.return_value = {}
+    provider_config.requires_session_configuration.return_value = True
+    provider_config.builds_setup_from_session_update.return_value = False
+    provider_config.session_configuration_request.return_value = setup
+    provider_config.unbilled_usage_on_session_close.return_value = None
+    provider_config.open_backend = AsyncMock(return_value=None)
+    provider_config.wrap_backend.side_effect = lambda opened: opened
+    handler = BaseLLMHTTPHandler()
+
+    with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=backend)):
+        await asyncio.wait_for(
+            handler.async_realtime(
+                model="amazon.nova-sonic-v1:0",
+                websocket=client_ws,
+                logging_obj=_realtime_logging_obj(),
+                provider_config=provider_config,
+                headers={},
+            ),
+            timeout=10,
+        )
+
+    assert backend.sent == [setup]
+    assert client_ws.sent == []
+
+
 class _JSONBodyAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
     def get_supported_openai_params(self, model):
         return []

tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py

@@ -1,3 +1,4 @@
+import base64
 import json
 from collections.abc import Mapping
 from typing import Final, cast
@@ -7,7 +8,7 @@ import pytest
 
 
 import litellm
-from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
+from litellm.llms.gemini.realtime.transformation import GEMINI_LIVE_DOCUMENTED_VOICES, GeminiRealtimeConfig
 from litellm.types.llms.gemini import BidiGenerateContentServerMessage
 
 
@@ -439,6 +440,51 @@ def test_gemini_realtime_session_update_with_tools():
     assert "parameters" in function_decl
 
 
+def _schema_nodes(schema: object):
+    if isinstance(schema, dict):
+        yield schema
+        for value in schema.values():
+            yield from _schema_nodes(value)
+    elif isinstance(schema, list):
+        for value in schema:
+            yield from _schema_nodes(value)
+
+
+def test_gemini_session_update_converts_flat_realtime_tool_schemas_to_gemini_schemas():
+    session_update: Final = {
+        "type": "session.update",
+        "session": {
+            "tools": [
+                {
+                    "type": "function",
+                    "name": "lookup_weather",
+                    "description": "Look up the weather in a city.",
+                    "parameters": {
+                        "type": "object",
+                        "$defs": {
+                            "City": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}
+                        },
+                        "properties": {"city": {"$ref": "#/$defs/City"}, "note": {"type": ["string", "null"]}},
+                        "required": ["city"],
+                    },
+                }
+            ]
+        },
+    }
+
+    messages: Final = GeminiRealtimeConfig().transform_realtime_request(
+        json.dumps(session_update), "gemini-2.5-flash", session_configuration_request=None
+    )
+
+    declaration: Final = json.loads(messages[0])["setup"]["tools"][0]["function_declarations"][0]
+    assert declaration["name"] == "lookup_weather"
+    parameters: Final = declaration["parameters"]
+    assert parameters["properties"]["city"]["properties"]["name"]["type"] == "string"
+    nodes: Final = tuple(_schema_nodes(parameters))
+    assert not any("$defs" in node or "$ref" in node for node in nodes)
+    assert all(isinstance(node["type"], str) for node in nodes if "type" in node)
+
+
 def test_gemini_session_update_defaults_to_audio_modality():
     config = GeminiRealtimeConfig()
 
@@ -613,6 +659,35 @@ def test_gemini_audio_only_live_models_drop_text_from_text_audio_combo(patch_gem
     assert setup["generationConfig"]["responseModalities"] == ["AUDIO"]
 
 
+@pytest.mark.parametrize("output_modalities", ["text", ["text"]])
+def test_gemini_session_update_accepts_a_single_string_output_modality(output_modalities):
+    """Regression: a bare string was iterated per character, so output_modalities
+    "text" reached Vertex as responseModalities ["T"], which it rejects with 1007."""
+    config = GeminiRealtimeConfig()
+    session_update = {"type": "session.update", "session": {"type": "realtime", "output_modalities": output_modalities}}
+
+    messages = config.transform_realtime_request(
+        json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
+    )
+
+    assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
+
+
+@pytest.mark.parametrize(
+    ("output_modalities", "expected"),
+    [(["video", "text"], ["TEXT"]), (["video"], ["AUDIO"]), (None, ["AUDIO"])],
+)
+def test_gemini_session_update_drops_modalities_live_cannot_produce(output_modalities, expected):
+    config = GeminiRealtimeConfig()
+    session_update = {"type": "session.update", "session": {"type": "realtime", "output_modalities": output_modalities}}
+
+    messages = config.transform_realtime_request(
+        json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
+    )
+
+    assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == expected
+
+
 def test_gemini_non_live_model_preserves_text_modality():
     config = GeminiRealtimeConfig()
     session_update = {
@@ -1396,22 +1471,71 @@ def test_gemini_input_audio_buffer_clear_is_local_noop():
     assert messages == []
 
 
-def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad():
+_MANUAL_VAD_SETUP = json.dumps({"setup": {"realtimeInputConfig": {"automaticActivityDetection": {"disabled": True}}}})
+_APPEND = json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(b"\x00\x01" * 80).decode()})
+
+
+def _realtime_input_keys(config: GeminiRealtimeConfig, client_event: str, setup: str | None) -> list[str]:
+    frames = config.transform_realtime_request(client_event, "gemini-live-2.5-flash-native-audio", setup)
+    return [key for frame in frames for key in json.loads(frame)["realtimeInput"]]
+
+
+@pytest.mark.parametrize("turn_end", ["input_audio_buffer.commit", "response.create"])
+def test_gemini_manual_vad_wraps_each_audio_turn_in_activity_start_and_end(turn_end):
+    """With activity detection off, Gemini Live answers only after activityEnd closes an
+    activityStart, and it closes the socket on any other shape of those markers."""
     config = GeminiRealtimeConfig()
-    setup = {
-        "setup": {
-            "realtimeInputConfig": {
-                "automaticActivityDetection": {"disabled": True},
-            }
-        }
-    }
-    messages = config.transform_realtime_request(
-        json.dumps({"type": "input_audio_buffer.commit"}),
-        "gemini-live-2.5-flash-native-audio",
-        session_configuration_request=json.dumps(setup),
+    end = json.dumps({"type": turn_end})
+
+    turns = [
+        [_realtime_input_keys(config, event, _MANUAL_VAD_SETUP) for event in (_APPEND, _APPEND, end, end)]
+        for _ in range(2)
+    ]
+
+    assert turns == [[["activityStart", "audio"], ["audio"], ["activityEnd"], []]] * 2
+
+
+def test_gemini_manual_vad_activity_markers_are_empty_messages():
+    config = GeminiRealtimeConfig()
+    commit = json.dumps({"type": "input_audio_buffer.commit"})
+
+    first_append = config.transform_realtime_request(_APPEND, "gemini-live-2.5-flash", _MANUAL_VAD_SETUP)
+    turn_end = config.transform_realtime_request(commit, "gemini-live-2.5-flash", _MANUAL_VAD_SETUP)
+
+    assert json.loads(first_append[0]) == {"realtimeInput": {"activityStart": {}}}
+    assert [json.loads(frame) for frame in turn_end] == [{"realtimeInput": {"activityEnd": {}}}]
+
+
+@pytest.mark.parametrize(
+    ("session", "expected_keys"),
+    [
+        ({"turn_detection": None}, [["activityStart", "audio"], ["activityEnd"]]),
+        ({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, [["activityStart", "audio"], ["activityEnd"]]),
+        ({"type": "realtime", "audio": {"input": {}}}, [["audio"], ["audioStreamEnd"]]),
+    ],
+)
+def test_gemini_null_turn_detection_makes_the_client_mark_each_turn(session, expected_keys):
+    """OpenAI's turn_detection null turns server VAD off, so the client commits each turn itself."""
+    config = GeminiRealtimeConfig()
+    setup = config.transform_realtime_request(
+        json.dumps({"type": "session.update", "session": session}), "gemini-live-2.5-flash-native-audio", None
     )
-    assert len(messages) == 1
-    assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": True}}
+    commit = json.dumps({"type": "input_audio_buffer.commit"})
+
+    keys = [_realtime_input_keys(config, event, setup[0]) for event in (_APPEND, commit)]
+
+    assert keys == expected_keys
+
+
+def test_gemini_automatic_vad_sends_no_activity_markers():
+    config = GeminiRealtimeConfig()
+    commit = json.dumps({"type": "input_audio_buffer.commit"})
+    response_create = json.dumps({"type": "response.create"})
+
+    keys = [_realtime_input_keys(config, event, None) for event in (_APPEND, commit)]
+
+    assert keys == [["audio"], ["audioStreamEnd"]]
+    assert config.transform_realtime_request(response_create, "gemini-live-2.5-flash", None) == []
 
 
 def test_gemini_tool_call_id_to_name_evicts_oldest_when_capped():
@@ -1883,13 +2007,75 @@ def test_map_openai_params_drops_stock_voice_case_insensitively():
     """
     cfg = GeminiRealtimeConfig()
 
-    dropped = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Alloy"})
+    dropped = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Alloy"}, model="gemini-3.8-live")
     assert "speechConfig" not in dropped.get("generationConfig", {})
 
-    passthrough = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Kore"})
+    passthrough = cfg.map_openai_params(
+        optional_params={}, non_default_params={"voice": "Kore"}, model="gemini-3.8-live"
+    )
     assert passthrough["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore"
 
 
+_DOCUMENTED_VOICE: Final = sorted(GEMINI_LIVE_DOCUMENTED_VOICES)[0].capitalize()
+
+
+@pytest.mark.parametrize("drop_params", [False, True])
+def test_google_ai_studio_forwards_a_documented_voice(drop_params):
+    config: Final = GeminiRealtimeConfig(drop_params=drop_params)
+    mapped: Final = config.map_openai_params(
+        optional_params={}, non_default_params={"voice": _DOCUMENTED_VOICE}, model="gemini-3.8-live"
+    )
+
+    speech_config: Final = mapped["generationConfig"]["speechConfig"]
+    assert speech_config["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == _DOCUMENTED_VOICE
+    assert config.setup_session_update_rejection("gemini-3.8-live", {"voice": _DOCUMENTED_VOICE}, setup_sent=False) is None
+
+
+@pytest.mark.parametrize(
+    ("session", "param"),
+    [
+        ({"voice": "Nova"}, "session.voice"),
+        ({"type": "realtime", "audio": {"output": {"voice": "Nova"}}}, "session.audio.output.voice"),
+    ],
+)
+def test_google_ai_studio_rejects_an_undocumented_voice_on_the_session_update(session, param):
+    rejection: Final = GeminiRealtimeConfig().setup_session_update_rejection(
+        "gemini-2.5-flash-native-audio-preview-12-2025", session, setup_sent=False
+    )
+
+    assert rejection is not None
+    assert (rejection.code, rejection.param) == ("invalid_value", param)
+
+
+def _setup_speech_config(config: GeminiRealtimeConfig, model: str, session: dict[str, object]) -> object:
+    messages: Final = config.transform_realtime_request(
+        json.dumps({"type": "session.update", "session": session}), model, session_configuration_request=None
+    )
+    return json.loads(messages[0])["setup"].get("generationConfig", {}).get("speechConfig")
+
+
+@pytest.mark.parametrize(
+    "session",
+    [{"voice": "Nova"}, {"type": "realtime", "audio": {"output": {"voice": "Nova"}}}],
+)
+def test_google_ai_studio_drops_an_undocumented_voice_with_drop_params(session):
+    config: Final = GeminiRealtimeConfig(drop_params=True)
+    model: Final = "gemini-2.5-flash-native-audio-preview-12-2025"
+
+    assert config.setup_session_update_rejection(model, session, setup_sent=False) is None
+    assert _setup_speech_config(config, model, session) is None
+
+
+@pytest.mark.parametrize("drop_params", [False, True])
+@pytest.mark.parametrize("model", ["gemini-3.8-live", "gemini-3.1-flash-live-preview"])
+def test_google_ai_studio_forwards_any_voice_to_models_that_accept_it(model, drop_params):
+    config: Final = GeminiRealtimeConfig(drop_params=drop_params)
+
+    assert config.setup_session_update_rejection(model, {"voice": "Nova"}, setup_sent=False) is None
+    speech_config: Final = _setup_speech_config(config, model, {"voice": "Nova"})
+    assert speech_config == {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Nova"}}}
+
+
 @pytest.fixture(autouse=False)
 def patch_gemini_transcribe_live_cost_map_entry(monkeypatch):
     """Inject the gemini-3.5-transcribe-live registry entry locally.
@@ -1954,6 +2140,55 @@ def test_gemini_transcribe_live_coerces_audio_modality_to_text(modalities, patch
     assert setup["generationConfig"]["responseModalities"] == ["TEXT"]
 
 
+@pytest.mark.parametrize("model", ["gemini-3.5-transcribe-live", "gemini/gemini-3.5-transcribe-live"])
+def test_gemini_transcribe_live_synthetic_session_created_reports_text_output(
+    model, patch_gemini_transcribe_live_cost_map_entry
+):
+    session = GeminiRealtimeConfig().transform_session_created_event(model, "sess_transcribe")["session"]
+
+    assert session["modalities"] == ["text"]
+
+
+@pytest.mark.parametrize(
+    ("model", "modalities", "producible"),
+    [
+        ("gemini-3.5-transcribe-live", ["audio"], False),
+        ("gemini-3.5-transcribe-live", ["audio", "text"], True),
+        ("gemini-live-2.5-flash-native-audio", ["text"], False),
+        ("gemini-live-2.5-flash-native-audio", ["text", "audio"], True),
+    ],
+)
+@pytest.mark.parametrize("drop_params", [False, True])
+def test_gemini_live_rejects_only_output_modalities_the_model_cannot_produce_at_all(
+    model,
+    modalities,
+    producible,
+    drop_params,
+    patch_gemini_transcribe_live_cost_map_entry,
+    patch_gemini_audio_cost_map_entries,
+):
+    rejection = GeminiRealtimeConfig(drop_params=drop_params).setup_session_update_rejection(
+        model, {"output_modalities": modalities}, setup_sent=False
+    )
+
+    assert (rejection is None) is (drop_params or producible)
+    assert rejection is None or rejection.param == "session.output_modalities"
+
+
+@pytest.mark.parametrize("drop_params", [False, True])
+def test_gemini_provider_realtime_config_carries_drop_params(drop_params):
+    from litellm.types.utils import LlmProviders
+    from litellm.utils import ProviderConfigManager
+
+    model = "gemini-live-2.5-flash-native-audio"
+    config = ProviderConfigManager.get_provider_realtime_config(
+        model=model, provider=LlmProviders.GEMINI, drop_params=drop_params
+    )
+
+    rejection = config.setup_session_update_rejection(model, {"modalities": ["text"]}, setup_sent=False)
+    assert (rejection is None) is drop_params
+
+
 def test_gemini_chat_model_with_text_output_modalities_keeps_audio_eager_setup(
     patch_gemini_transcribe_live_cost_map_entry,
 ):
@@ -2169,6 +2404,47 @@ def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_tra
     assert config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") is None
 
 
+def test_gemini_realtime_two_output_modalities_collapse_to_audio():
+    """Gemini Live closes the socket with 1007 when the setup names two response
+    modalities, so a GA session.update asking for ["text", "audio"] must reach the
+    backend as AUDIO only, the same choice the beta shim makes."""
+    config = GeminiRealtimeConfig()
+    session_update = {
+        "type": "session.update",
+        "session": {"output_modalities": ["text", "audio"], "instructions": "Be brief."},
+    }
+
+    messages = config.transform_realtime_request(
+        json.dumps(session_update),
+        "gemini-live-2.5-flash",
+        session_configuration_request=None,
+    )
+
+    assert len(messages) == 1
+    assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"]
+
+
+def test_gemini_realtime_inf_max_response_output_tokens_is_not_forwarded():
+    """OpenAI's default max_response_output_tokens is the string "inf", which Gemini
+    rejects with 1007 (TYPE_INT32), so only integer limits map to maxOutputTokens."""
+    config = GeminiRealtimeConfig()
+
+    def generation_config_for(limit):
+        session_update = {
+            "type": "session.update",
+            "session": {"modalities": ["text"], "max_response_output_tokens": limit},
+        }
+        messages = config.transform_realtime_request(
+            json.dumps(session_update),
+            "gemini-live-2.5-flash",
+            session_configuration_request=None,
+        )
+        return json.loads(messages[0])["setup"]["generationConfig"]
+
+    assert "maxOutputTokens" not in generation_config_for("inf")
+    assert generation_config_for(200)["maxOutputTokens"] == 200
+
+
 def _grounded_live_frame(grounding_metadata: Mapping[str, object] | None) -> Mapping[str, object]:
     """One Live server frame. Grounding metadata and usageMetadata arrive together, as Vertex sends them."""
     from typing import Final

tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py

@@ -343,6 +343,26 @@ def test_body_carries_no_aws_credentials(
     assert client.request_headers["X-Amz-Security-Token"] == "test-session-token"
 
 
+@pytest.mark.parametrize("model", ["sagemaker_chat/my-endpoint", "sagemaker_nova/my-endpoint"])
+def test_inference_component_travels_only_in_the_header(monkeypatch: pytest.MonkeyPatch, model: str):
+    client = _invoke_sagemaker_chat(monkeypatch, model=model, model_id="my-inference-component")
+
+    assert client.request_headers[_INFERENCE_COMPONENT_HEADER] == "my-inference-component"
+    assert "model_id" not in client.request_body
+
+
+def test_inference_component_header_keeps_header_keys_that_are_not_strings():
+    headers = SagemakerChatConfig().validate_environment(
+        headers={7: "other", "x-amzn-sagemaker-inference-component": "client-component"},
+        model="my-endpoint",
+        messages=[],
+        optional_params={"model_id": "my-inference-component"},
+        litellm_params={},
+    )
+
+    assert headers == {7: "other", _INFERENCE_COMPONENT_HEADER: "my-inference-component"}
+
+
 def test_body_model_stays_the_endpoint_name_when_hf_model_name_is_unset(monkeypatch):
     """Without `hf_model_name` the body must keep the model it has today."""
     client = _invoke_sagemaker_chat(monkeypatch)

tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py

@@ -372,6 +372,27 @@ def test_completion_sagemaker_non_stream(
     assert response.choices[0].message.content == "SageMaker reply"
 
 
+@pytest.mark.usefixtures("httpx_transport")
+@pytest.mark.usefixtures("fake_provider_credentials")
+def test_completion_sagemaker_sends_the_inference_component_only_as_a_header(respx_mock: respx.MockRouter) -> None:
+    url: Final = "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/my-endpoint/invocations"
+    route: Final = respx_mock.post(url).mock(
+        return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"})
+    )
+    litellm.completion(
+        model="sagemaker/my-endpoint",
+        messages=[{"role": "user", "content": "hi"}],
+        temperature=0.2,
+        model_id="my-inference-component",
+        aws_region_name="us-west-2",
+        num_retries=0,
+    )
+
+    request: Final = route.calls[0].request
+    assert request.headers["X-Amzn-SageMaker-Inference-Component"] == "my-inference-component"
+    assert json.loads(request.content) == {"inputs": "hi", "parameters": {"temperature": 0.2}}
+
+
 @pytest.mark.usefixtures("httpx_transport")
 @pytest.mark.usefixtures("fake_provider_credentials")
 def test_completion_sagemaker_non_stream_with_aws_params(respx_mock: respx.MockRouter) -> None:

tests/unit/llms/test_subscription_accounts.py

@@ -0,0 +1,110 @@
+import base64
+import json
+from pathlib import Path
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.constants import SUBSCRIPTION_BACKED_PROVIDERS
+from litellm.llms.chatgpt.common_utils import OPENAI_AUTH_CLAIM_KEY
+from litellm.llms.subscription_accounts import SUBSCRIPTION_ACCOUNT_ID_RESOLVERS, resolve_subscription_account_id
+from litellm.types.utils import CredentialItem
+
+
+def _write_login(path: Path, account_id: str) -> Path:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps({"account_id": account_id}))
+    return path
+
+
+def _access_token_for(account_id: str) -> str:
+    claims: Final = json.dumps({OPENAI_AUTH_CLAIM_KEY: {"chatgpt_account_id": account_id}}).encode()
+    payload: Final = base64.urlsafe_b64encode(claims).decode().rstrip("=")
+    return f"header.{payload}.signature"
+
+
+@pytest.fixture
+def signed_in_chatgpt_token_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
+    monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False)
+    _write_login(tmp_path / "auth.json", "acct-from-token-file")
+    return tmp_path
+
+
+def test_every_subscription_backed_provider_has_an_account_id_resolver() -> None:
+    assert frozenset(SUBSCRIPTION_ACCOUNT_ID_RESOLVERS) == SUBSCRIPTION_BACKED_PROVIDERS
+
+
+def test_chatgpt_resolves_to_the_account_in_the_signed_in_token_file(signed_in_chatgpt_token_dir: Path) -> None:
+    assert resolve_subscription_account_id("chatgpt", {"model": "chatgpt/gpt-5.6-terra"}) == "acct-from-token-file"
+
+
+def test_each_deployment_takes_the_account_of_its_own_auth_file(signed_in_chatgpt_token_dir: Path) -> None:
+    first: Final = _write_login(signed_in_chatgpt_token_dir / "team-a" / "auth.json", "acct-team-a")
+    second: Final = _write_login(signed_in_chatgpt_token_dir / "team-b" / "auth.json", "acct-team-b")
+
+    assert resolve_subscription_account_id("chatgpt", {"chatgpt_auth_file": str(first)}) == "acct-team-a"
+    assert resolve_subscription_account_id("chatgpt", {"chatgpt_auth_file": str(second)}) == "acct-team-b"
+
+
+def test_a_configured_access_token_resolves_to_its_own_account(signed_in_chatgpt_token_dir: Path) -> None:
+    litellm_params: Final = {"model": "chatgpt/gpt-5.6-terra", "api_key": _access_token_for("acct-from-api-key")}
+
+    assert resolve_subscription_account_id("chatgpt", litellm_params) == "acct-from-api-key"
+
+
+def test_a_metered_provider_resolves_to_no_account_even_with_a_chatgpt_login(
+    signed_in_chatgpt_token_dir: Path,
+) -> None:
+    assert resolve_subscription_account_id("openai", {"model": "openai/gpt-5.6-terra"}) is None
+
+
+def test_chatgpt_without_a_login_resolves_to_no_account(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    empty_dir: Final = tmp_path / "no-login"
+    monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(empty_dir))
+    monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False)
+    assert resolve_subscription_account_id("chatgpt", {"model": "chatgpt/gpt-5.6-terra"}) is None
+
+
+def test_resolving_an_account_leaves_the_login_file_untouched(signed_in_chatgpt_token_dir: Path) -> None:
+    login: Final = signed_in_chatgpt_token_dir / "no-saved-account" / "auth.json"
+    login.parent.mkdir(parents=True)
+    saved: Final = json.dumps({"access_token": _access_token_for("acct-derived"), "refresh_token": "rt-current"})
+    login.write_text(saved)
+
+    assert resolve_subscription_account_id("chatgpt", {"chatgpt_auth_file": str(login)}) == "acct-derived"
+    assert login.read_text() == saved
+
+
+@pytest.mark.parametrize("credential_field", ["chatgpt_auth_file", "api_key"])
+def test_a_named_credential_resolves_to_the_account_it_signs_in_as(
+    credential_field: str, signed_in_chatgpt_token_dir: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    login: Final = _write_login(signed_in_chatgpt_token_dir / "named" / "auth.json", "acct-named-credential")
+    values: Final = {
+        "chatgpt_auth_file": {"chatgpt_auth_file": str(login)},
+        "api_key": {"api_key": _access_token_for("acct-named-credential")},
+    }[credential_field]
+    monkeypatch.setattr(
+        litellm,
+        "credential_list",
+        [CredentialItem(credential_name="chatgpt-team", credential_values=values, credential_info={})],
+    )
+    litellm_params: Final = {"model": "chatgpt/gpt-5.6-terra", "litellm_credential_name": "chatgpt-team"}
+
+    assert resolve_subscription_account_id("chatgpt", litellm_params) == "acct-named-credential"
+
+
+@pytest.mark.parametrize(
+    ("api_key", "expected_account"),
+    [("oidc/env/CHATGPT_TEAM_TOKEN", "acct-from-oidc-env"), ("oidc/env/CHATGPT_UNSET_TOKEN", None)],
+)
+def test_an_oidc_env_api_key_resolves_to_the_account_of_the_token_it_points_to(
+    api_key: str, expected_account: str | None, signed_in_chatgpt_token_dir: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setenv("CHATGPT_TEAM_TOKEN", _access_token_for("acct-from-oidc-env"))
+    monkeypatch.delenv("CHATGPT_UNSET_TOKEN", raising=False)
+    litellm_params: Final = {"model": "chatgpt/gpt-5.6-terra", "api_key": api_key}
+
+    assert resolve_subscription_account_id("chatgpt", litellm_params) == expected_account

tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py

@@ -18,6 +18,7 @@ import websockets.exceptions  # registers websockets.exceptions on the websocket
 
 import litellm
 from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
+from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES
 from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig
 
 # ---------------------------------------------------------------------------
@@ -179,6 +180,25 @@ def test_vertex_audio_only_live_model_coerces_text_modality_to_audio(
     assert setup["generationConfig"]["responseModalities"] == ["AUDIO"]
 
 
+@pytest.mark.parametrize("drop_params", [False, True])
+def test_vertex_realtime_config_carries_drop_params(drop_params, patch_native_audio_cost_map_entry):
+    from litellm.llms.vertex_ai.realtime.transformation import vertex_realtime_config
+
+    cfg: Final = vertex_realtime_config(
+        _NATIVE_AUDIO_MODEL,
+        access_token="tok",
+        resolve_access_token=AsyncMock(return_value="tok"),
+        project="my-proj",
+        location="us-central1",
+        drop_params=drop_params,
+    )
+
+    rejection: Final = cfg.setup_session_update_rejection(
+        _NATIVE_AUDIO_MODEL, {"modalities": ["text"]}, setup_sent=False
+    )
+    assert (rejection is None) is drop_params
+
+
 def test_vertex_session_update_normalizes_ga_remapped_fields(
     patch_native_audio_cost_map_entry,
 ):
@@ -253,6 +273,26 @@ def test_vertex_keeps_turn_detection_tuning_when_auto_response_is_off():
     }
 
 
+
+@pytest.mark.parametrize(
+    ("session", "disabled"),
+    [
+        ({"turn_detection": None}, True),
+        ({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, True),
+        ({"type": "realtime", "audio": {"input": {}}}, False),
+    ],
+)
+def test_vertex_turns_activity_detection_off_for_null_turn_detection(session, disabled):
+    """OpenAI's turn_detection null is push-to-talk, so Vertex must wait for the client's activity markers."""
+    cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1")
+    messages: Final = cfg.transform_realtime_request(
+        json.dumps({"type": "session.update", "session": session}),
+        _NATIVE_AUDIO_MODEL,
+        session_configuration_request=None,
+    )
+    setup: Final = json.loads(messages[0])["setup"]
+    assert setup["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] is disabled
+
 # ---------------------------------------------------------------------------
 # Round-trip test: text-in / text-out via RealTimeStreaming
 # ---------------------------------------------------------------------------
@@ -607,3 +647,73 @@ def test_vertex_native_audio_unmapped_voice_passes_through(patch_native_audio_co
 
     generation_config = json.loads(messages[0])["setup"]["generationConfig"]
     assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore"
+
+
+def test_vertex_session_update_two_output_modalities_collapse_to_audio():
+    """Vertex Live rejects a setup naming two response modalities, so a GA
+    ["text", "audio"] request must be sent as AUDIO only."""
+    cfg = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="global")
+    session_update = {
+        "type": "session.update",
+        "session": {"output_modalities": ["text", "audio"], "instructions": "Be brief."},
+    }
+
+    messages = cfg.transform_realtime_request(
+        json.dumps(session_update),
+        "gemini-live-2.5-flash",
+        session_configuration_request=None,
+    )
+
+    assert len(messages) == 1
+    assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"]
+
+
+@pytest.mark.parametrize(
+    ("session", "param"),
+    [
+        ({"voice": "Nova"}, "session.voice"),
+        ({"type": "realtime", "audio": {"output": {"voice": "Nova"}}}, "session.audio.output.voice"),
+    ],
+)
+def test_vertex_rejects_an_undocumented_voice_on_the_session_update(session, param):
+    cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1")
+
+    rejection: Final = cfg.setup_session_update_rejection("gemini-live-2.5-flash", session, setup_sent=False)
+
+    assert rejection is not None
+    assert (rejection.code, rejection.param) == ("invalid_value", param)
+
+
+@pytest.mark.parametrize("voice", ["Kore", "alloy"])
+def test_vertex_accepts_documented_and_openai_stock_voices_on_the_session_update(voice):
+    cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1")
+
+    assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", {"voice": voice}, setup_sent=False) is None
+
+
+def test_vertex_drops_an_undocumented_voice_with_drop_params():
+    cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1", drop_params=True)
+    session_update: Final = {"type": "session.update", "session": {"voice": "Nova", "instructions": "Be brief."}}
+
+    assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None
+    messages: Final = cfg.transform_realtime_request(
+        json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
+    )
+    assert "speechConfig" not in json.loads(messages[0])["setup"].get("generationConfig", {})
+
+
+@pytest.mark.parametrize("drop_params", [False, True])
+@pytest.mark.parametrize("voice", ["fable", "onyx"])
+def test_vertex_sends_the_documented_voice_an_openai_tts_voice_maps_to(voice, drop_params):
+    cfg: Final = VertexAIRealtimeConfig(
+        access_token="tok", project="my-proj", location="us-central1", drop_params=drop_params
+    )
+    session_update: Final = {"type": "session.update", "session": {"voice": voice}}
+
+    assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None
+    messages: Final = cfg.transform_realtime_request(
+        json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
+    )
+    assert json.loads(messages[0])["setup"]["generationConfig"]["speechConfig"] == {
+        "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES[voice]}}
+    }

tests/unit/proxy/auth/test_handle_jwt.py

@@ -913,6 +913,31 @@ async def test_sync_user_role_and_teams_cache_invalidation_on_team_change():
     assert call_kwargs.kwargs["model_type"] == LiteLLM_UserTable
 
 
+@pytest.mark.asyncio
+async def test_sync_user_role_and_teams_keeps_record_order_and_adds_claim_teams_in_claim_order():
+    jwt_handler: Final = JWTHandler()
+    jwt_handler.update_environment(
+        prisma_client=None,
+        user_api_key_cache=AsyncMock(),
+        litellm_jwtauth=LiteLLM_JWTAuth(team_ids_jwt_field="groups", sync_user_role_and_teams=True),
+    )
+    cache: Final = AsyncMock()
+    user: Final = LiteLLM_UserTable(user_id="u1", teams=["team-zeta", "team-kappa", "team-alpha"])
+    token: Final = {"groups": ["team-mu", "team-alpha", "team-beta", "team-mu", "team-zeta", "team-omega"]}
+
+    with patch(
+        "litellm.proxy.management_endpoints.scim.scim_v2.patch_team_membership",
+        new_callable=AsyncMock,
+    ) as mock_patch:
+        await JWTAuthManager.sync_user_role_and_teams(jwt_handler, token, user, AsyncMock(), user_api_key_cache=cache)
+
+    assert mock_patch.await_args.kwargs["teams_ids_to_add_user_to"] == ["team-mu", "team-beta", "team-omega"]
+    assert mock_patch.await_args.kwargs["teams_ids_to_remove_user_from"] == ["team-kappa"]
+    expected_teams: Final = ["team-zeta", "team-alpha", "team-mu", "team-beta", "team-omega"]
+    assert user.teams == expected_teams
+    assert cache.async_set_cache.await_args.kwargs["value"].teams == expected_teams
+
+
 @pytest.mark.asyncio
 async def test_sync_user_role_and_teams_no_cache_write_when_nothing_changes():
     """Test that cache is NOT written when role and teams already match."""

tests/unit/proxy/common_utils/test_config_sync_pubsub.py

@@ -5,6 +5,7 @@ from typing import Callable, Coroutine, Final, Iterable, List, Optional, Tuple
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
+from pydantic import BaseModel, ConfigDict
 from redis.asyncio import Redis
 
 import litellm
@@ -102,8 +103,8 @@ class _ScriptedPubSubClient:
 
 
 class _QueuePubSub:
-    def __init__(self, initial_messages: Iterable[str] = ()) -> None:
-        self.queue: "asyncio.Queue[str]" = asyncio.Queue()
+    def __init__(self, initial_messages: Iterable[object] = ()) -> None:
+        self.queue: "asyncio.Queue[object]" = asyncio.Queue()
         for message in initial_messages:
             self.queue.put_nowait(message)
         self.subscribed_channels: List[str] = []
@@ -112,7 +113,7 @@ class _QueuePubSub:
     async def subscribe(self, *channels: str) -> None:
         self.subscribed_channels.extend(channels)
 
-    async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]:
+    async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]:
         if timeout == 0:
             try:
                 return self.queue.get_nowait()
@@ -128,7 +129,7 @@ class _QueuePubSub:
 
 
 class _BrokenPubSub(_QueuePubSub):
-    async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]:
+    async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]:
         raise ConnectionError("connection lost")
 
 
@@ -138,11 +139,11 @@ class _CloseFailingBrokenPubSub(_BrokenPubSub):
 
 
 class _EmptyPollsThenMessagePubSub(_QueuePubSub):
-    def __init__(self, empty_polls: int, initial_messages: Iterable[str] = ()) -> None:
+    def __init__(self, empty_polls: int, initial_messages: Iterable[object] = ()) -> None:
         super().__init__(initial_messages=initial_messages)
         self.remaining_empty_polls = empty_polls
 
-    async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]:
+    async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]:
         if timeout != 0 and self.remaining_empty_polls > 0:
             self.remaining_empty_polls -= 1
             return None
@@ -157,6 +158,38 @@ class _FakeClock:
         return self.now
 
 
+class _VirtualTimePubSub(_QueuePubSub):
+    """Timed polls move the injected clock forward instead of waiting on the wall clock.
+
+    A delivery scheduled at a clock time is returned by the first poll whose window
+    reaches it, with the clock set to that time, so a test asserts when each resync
+    and targeted apply ran in subscriber time.
+    """
+
+    def __init__(self, clock: _FakeClock, deliveries: Iterable[Tuple[float, object]] = ()) -> None:
+        super().__init__()
+        self._clock = clock
+        self.pending: List[Tuple[float, object]] = sorted(deliveries, key=lambda delivery: delivery[0])
+
+    async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]:
+        await asyncio.sleep(0)
+        due_by = self._clock.now + timeout
+        if self.pending and self.pending[0][0] <= due_by:
+            delivered_at, message = self.pending.pop(0)
+            self._clock.now = max(self._clock.now, delivered_at)
+            return message
+        self._clock.now = due_by
+        return None
+
+
+def _change(*model_ids: str) -> str:
+    return json.dumps({"object_type": "litellm_proxymodeltable", "model_ids": list(model_ids)})
+
+
+def _redis_delivery(message: str) -> dict:
+    return {"type": "message", "pattern": None, "channel": b"litellm_proxy.config_change", "data": message.encode()}
+
+
 class _ScriptedPubSubRedisClient(Redis):
     def __init__(self, pubsubs: Iterable[_QueuePubSub]) -> None:
         self._scripted_pubsubs = iter(pubsubs)
@@ -204,7 +237,16 @@ async def test_publish_sends_object_type_json_on_channel() -> None:
     assert len(client.published) == 1
     channel, message = client.published[0]
     assert channel == "litellm_proxy.config_change"
-    assert json.loads(message) == {"object_type": "litellm_proxymodeltable"}
+    assert json.loads(message) == {"object_type": "litellm_proxymodeltable", "model_ids": []}
+
+
+async def test_publish_carries_the_written_model_ids() -> None:
+    client = _RecordingRedisClient()
+    cache = _FakeRedisCache(client)
+
+    await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable", model_ids=("m-1", "m-2"))
+
+    assert json.loads(client.published[0][1]) == {"object_type": "litellm_proxymodeltable", "model_ids": ["m-1", "m-2"]}
 
 
 async def test_publish_uses_namespaced_channel() -> None:
@@ -235,7 +277,7 @@ async def test_publish_reaches_cluster_derived_pubsub_clients() -> None:
     await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable")
 
     assert client.published == [
-        (CONFIG_SYNC_CHANNEL, json.dumps({"object_type": "litellm_proxymodeltable"}))
+        (CONFIG_SYNC_CHANNEL, json.dumps({"object_type": "litellm_proxymodeltable", "model_ids": []}))
     ]
 
 
@@ -312,29 +354,210 @@ class _MaxJitterRandom(random.Random):
         return b
 
 
-async def test_debounce_sleep_adds_jitter_from_injected_rng() -> None:
-    pubsub = _QueuePubSub(initial_messages=[json.dumps({"object_type": "litellm_proxymodeltable"})])
-    cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub]))
-    sleeps: List[float] = []
-    fired = asyncio.Event()
+def _virtual_time_subscriber(
+    pubsub: _VirtualTimePubSub,
+    clock: _FakeClock,
+    events: List[str],
+    resynced: asyncio.Event,
+    *,
+    debounce_seconds: float = 0.0,
+    jitter_max_seconds: float = 0.0,
+    min_resync_interval_seconds: float = 10.0,
+    rng: Optional[random.Random] = None,
+    expected_resyncs: int = 1,
+    apply_error: Optional[Exception] = None,
+) -> ConfigSyncSubscriber:
+    async def resync() -> None:
+        events.append(f"resync@{clock.now}")
+        if sum(event.startswith("resync@") for event in events) >= expected_resyncs:
+            resynced.set()
 
-    async def recording_sleep(seconds: float) -> None:
-        sleeps.append(seconds)
+    async def apply(model_ids: Tuple[str, ...]) -> None:
+        events.append(f"apply:{','.join(model_ids)}@{clock.now}")
+        if apply_error is not None:
+            raise apply_error
 
-    subscriber = ConfigSyncSubscriber(
-        redis_cache=cache,
-        resync_callbacks=(_recording_callback([], "resync", fired),),
-        debounce_seconds=1.0,
-        jitter_max_seconds=4.0,
-        rng=_MaxJitterRandom(),
-        sleep=recording_sleep,
+    return ConfigSyncSubscriber(
+        redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])),
+        resync_callbacks=(resync,),
+        apply_model_ids=apply,
+        debounce_seconds=debounce_seconds,
+        jitter_max_seconds=jitter_max_seconds,
+        min_resync_interval_seconds=min_resync_interval_seconds,
+        rng=rng,
+        monotonic=clock,
     )
 
-    subscriber.start()
-    await asyncio.wait_for(fired.wait(), timeout=5)
-    await subscriber.stop()
 
-    assert sleeps == [5.0]
+async def _run_until_resynced(subscriber: ConfigSyncSubscriber, resynced: asyncio.Event) -> None:
+    subscriber.start()
+    try:
+        await asyncio.wait_for(resynced.wait(), timeout=5)
+    finally:
+        await subscriber.stop()
+
+
+async def test_debounce_waits_out_the_jitter_from_injected_rng() -> None:
+    clock = _FakeClock()
+    pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change())])
+    events: List[str] = []
+    resynced = asyncio.Event()
+    subscriber = _virtual_time_subscriber(
+        pubsub, clock, events, resynced, debounce_seconds=1.0, jitter_max_seconds=4.0, rng=_MaxJitterRandom()
+    )
+
+    await _run_until_resynced(subscriber, resynced)
+
+    assert events == ["resync@1005.0"]
+
+
+async def test_published_model_ids_apply_on_receipt_while_the_full_resync_still_debounces() -> None:
+    clock = _FakeClock()
+    pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change("m-1"))])
+    events: List[str] = []
+    resynced = asyncio.Event()
+    subscriber = _virtual_time_subscriber(
+        pubsub, clock, events, resynced, debounce_seconds=1.0, jitter_max_seconds=4.0, rng=_MaxJitterRandom()
+    )
+
+    await _run_until_resynced(subscriber, resynced)
+
+    assert events == ["apply:m-1@1000.0", "resync@1005.0"]
+
+
+async def test_model_ids_published_inside_the_throttle_wait_apply_without_waiting() -> None:
+    clock = _FakeClock()
+    pubsub = _VirtualTimePubSub(
+        clock, deliveries=[(1000.0, _change()), (1004.0, _change("m-2")), (1006.0, _change("m-3", "m-4"))]
+    )
+    events: List[str] = []
+    resynced = asyncio.Event()
+    subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2)
+
+    await _run_until_resynced(subscriber, resynced)
+
+    assert events == ["resync@1000.0", "apply:m-2@1004.0", "apply:m-3,m-4@1006.0", "resync@1010.0"]
+
+
+async def test_model_ids_already_waiting_apply_together_in_one_call() -> None:
+    clock = _FakeClock()
+    pubsub = _VirtualTimePubSub(
+        clock, deliveries=[(1000.0, _change("m-1")), (1000.0, _change("m-2", "m-1")), (1000.0, _change("m-3"))]
+    )
+    events: List[str] = []
+    resynced = asyncio.Event()
+    subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced)
+
+    await _run_until_resynced(subscriber, resynced)
+
+    assert events == ["apply:m-1,m-2,m-3@1000.0", "resync@1000.0"]
+
+
+async def test_redis_shaped_message_applies_its_model_ids() -> None:
+    clock = _FakeClock()
+    pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _redis_delivery(_change("m-1")))])
+    events: List[str] = []
+    resynced = asyncio.Event()
+    subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced)
+
+    await _run_until_resynced(subscriber, resynced)
+
+    assert events == ["apply:m-1@1000.0", "resync@1000.0"]
+
+
+async def test_messages_without_usable_model_ids_only_schedule_the_full_resync() -> None:
+    clock = _FakeClock()
+    pubsub = _VirtualTimePubSub(
+        clock,
+        deliveries=[
+            (1000.0, "change"),
+            (1004.0, json.dumps({"object_type": "litellm_proxymodeltable"})),
+            (1005.0, json.dumps({"object_type": "litellm_proxymodeltable", "model_ids": "m-1"})),
+            (1006.0, {"type": "message", "pattern": None, "channel": b"c", "data": 7}),
+            (1007.0, None),
+        ],
+    )
+    events: List[str] = []
+    resynced = asyncio.Event()
+    subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2)
+
+    await _run_until_resynced(subscriber, resynced)
+
+    assert events == ["resync@1000.0", "resync@1010.0"]
+
+
+async def test_failing_targeted_apply_still_runs_the_full_resync() -> None:
+    clock = _FakeClock()
+    pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change("m-1"))])
+    events: List[str] = []
+    resynced = asyncio.Event()
+    subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, apply_error=RuntimeError("db down"))
+
+    await _run_until_resynced(subscriber, resynced)
+
+    assert events == ["apply:m-1@1000.0", "resync@1000.0"]
+
+
+async def test_subscriber_without_an_apply_callback_only_resyncs() -> None:
+    clock = _FakeClock()
+    pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change("m-1"))])
+    resyncs: List[str] = []
+    fired = asyncio.Event()
+    subscriber = ConfigSyncSubscriber(
+        redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])),
+        resync_callbacks=(_recording_callback(resyncs, "resync", fired),),
+        debounce_seconds=0.0,
+        jitter_max_seconds=0.0,
+        monotonic=clock,
+    )
+
+    await _run_until_resynced(subscriber, fired)
+
+    assert resyncs == ["resync"]
+
+
+async def test_peer_holds_every_published_row_before_its_next_full_resync() -> None:
+    publisher = _RecordingRedisClient()
+    cache = _FakeRedisCache(publisher)
+    await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable", model_ids=("m-order-1",))
+    await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable", model_ids=("m-order-2",))
+    on_the_wire = [_redis_delivery(message) for _, message in publisher.published]
+    clock = _FakeClock()
+    pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, on_the_wire[0]), (1007.0, on_the_wire[1])])
+    db_rows_written_at = {"m-order-1": 1000.0, "m-order-2": 1007.0}
+    registry: set = set()
+    snapshots: List[Tuple[str, float, frozenset]] = []
+    resynced = asyncio.Event()
+
+    async def apply(model_ids: Tuple[str, ...]) -> None:
+        registry.update(model_ids)
+        snapshots.append(("apply", clock.now, frozenset(registry)))
+
+    async def resync() -> None:
+        registry.update(row for row, written_at in db_rows_written_at.items() if written_at <= clock.now)
+        snapshots.append(("resync", clock.now, frozenset(registry)))
+        if sum(kind == "resync" for kind, _, _ in snapshots) == 2:
+            resynced.set()
+
+    subscriber = ConfigSyncSubscriber(
+        redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])),
+        resync_callbacks=(resync,),
+        apply_model_ids=apply,
+        debounce_seconds=1.0,
+        jitter_max_seconds=5.0,
+        min_resync_interval_seconds=10.0,
+        rng=_MaxJitterRandom(),
+        monotonic=clock,
+    )
+
+    await _run_until_resynced(subscriber, resynced)
+
+    assert snapshots == [
+        ("apply", 1000.0, frozenset({"m-order-1"})),
+        ("resync", 1006.0, frozenset({"m-order-1"})),
+        ("apply", 1007.0, frozenset({"m-order-1", "m-order-2"})),
+        ("resync", 1016.0, frozenset({"m-order-1", "m-order-2"})),
+    ]
 
 
 def test_default_jitter_window_is_nonzero() -> None:
@@ -345,92 +568,43 @@ def test_default_min_resync_interval_caps_reload_rate() -> None:
     assert CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS > CONFIG_SYNC_JITTER_MAX_SECONDS
 
 
-def _throttled_subscriber(
-    cache: object,
-    events: List[str],
-    fired: asyncio.Event,
-    clock: _FakeClock,
-    min_resync_interval_seconds: float = 10.0,
-) -> ConfigSyncSubscriber:
-    async def recording_sleep(seconds: float) -> None:
-        events.append(f"sleep:{seconds}")
-        await asyncio.sleep(0)
-
-    async def resync() -> None:
-        events.append("resync")
-        fired.set()
-
-    return ConfigSyncSubscriber(
-        redis_cache=cache,
-        resync_callbacks=(resync,),
-        debounce_seconds=0.0,
-        jitter_max_seconds=0.0,
-        min_resync_interval_seconds=min_resync_interval_seconds,
-        sleep=recording_sleep,
-        monotonic=clock,
-    )
-
-
 async def test_resync_arriving_inside_min_interval_waits_out_the_remainder() -> None:
-    pubsub = _QueuePubSub()
-    cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub]))
-    events: List[str] = []
-    fired = asyncio.Event()
     clock = _FakeClock()
-    subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock)
+    pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, "change"), (1004.0, "change")])
+    events: List[str] = []
+    resynced = asyncio.Event()
+    subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2)
 
-    subscriber.start()
-    pubsub.queue.put_nowait("change")
-    await asyncio.wait_for(fired.wait(), timeout=5)
-    fired.clear()
-    clock.now += 4.0
-    pubsub.queue.put_nowait("change")
-    await asyncio.wait_for(fired.wait(), timeout=5)
-    await subscriber.stop()
+    await _run_until_resynced(subscriber, resynced)
 
-    assert events == ["sleep:0.0", "resync", "sleep:0.0", "sleep:6.0", "resync"]
+    assert events == ["resync@1000.0", "resync@1010.0"]
 
 
 async def test_resync_after_min_interval_elapsed_is_not_throttled() -> None:
-    pubsub = _QueuePubSub()
-    cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub]))
-    events: List[str] = []
-    fired = asyncio.Event()
     clock = _FakeClock()
-    subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock)
+    pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, "change"), (1030.0, "change")])
+    events: List[str] = []
+    resynced = asyncio.Event()
+    subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2)
 
-    subscriber.start()
-    pubsub.queue.put_nowait("change")
-    await asyncio.wait_for(fired.wait(), timeout=5)
-    fired.clear()
-    clock.now += 30.0
-    pubsub.queue.put_nowait("change")
-    await asyncio.wait_for(fired.wait(), timeout=5)
-    await subscriber.stop()
+    await _run_until_resynced(subscriber, resynced)
 
-    assert events == ["sleep:0.0", "resync", "sleep:0.0", "resync"]
+    assert events == ["resync@1000.0", "resync@1030.0"]
 
 
 async def test_writes_during_the_throttle_wait_collapse_into_the_next_resync() -> None:
-    pubsub = _QueuePubSub()
-    cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub]))
-    events: List[str] = []
-    fired = asyncio.Event()
     clock = _FakeClock()
-    subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock)
+    pubsub = _VirtualTimePubSub(
+        clock, deliveries=[(1000.0, "change")] + [(1004.0 + offset, "change") for offset in range(5)]
+    )
+    events: List[str] = []
+    resynced = asyncio.Event()
+    subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2)
 
-    subscriber.start()
-    pubsub.queue.put_nowait("change")
-    await asyncio.wait_for(fired.wait(), timeout=5)
-    fired.clear()
-    for _ in range(5):
-        pubsub.queue.put_nowait("change")
-    await asyncio.wait_for(fired.wait(), timeout=5)
-    await asyncio.sleep(0.1)
-    await subscriber.stop()
+    await _run_until_resynced(subscriber, resynced)
 
-    assert events.count("resync") == 2
-    assert pubsub.queue.empty()
+    assert events == ["resync@1000.0", "resync@1010.0"]
+    assert pubsub.pending == []
 
 
 async def test_polls_without_messages_do_not_trigger_resyncs() -> None:
@@ -646,13 +820,39 @@ class _AllWritesTableActions:
         return action
 
 
-def _recording_publish(calls: List[Tuple[str, str]]) -> Callable[[str], Coroutine[None, None, None]]:
-    async def publish(object_type: str) -> None:
+def _recording_publish(calls: List[Tuple[str, str]]) -> Callable[[str, Tuple[str, ...]], Coroutine[None, None, None]]:
+    async def publish(object_type: str, model_ids: Tuple[str, ...]) -> None:
         calls.append(("publish", object_type))
 
     return publish
 
 
+def _recording_publish_with_ids(
+    calls: List[Tuple[str, Tuple[str, ...]]],
+) -> Callable[[str, Tuple[str, ...]], Coroutine[None, None, None]]:
+    async def publish(object_type: str, model_ids: Tuple[str, ...]) -> None:
+        calls.append((object_type, model_ids))
+
+    return publish
+
+
+class _ModelRow(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model_id: str
+    model_name: str
+
+
+class _ModelRowTableActions:
+    def __getattr__(self, name: str) -> Callable[..., Coroutine[None, None, object]]:
+        async def action(*args: object, **kwargs: object) -> object:
+            if name.endswith("_many"):
+                return 2
+            return _ModelRow(model_id="m-1", model_name="gpt-5.2")
+
+        return action
+
+
 def test_wrapper_passes_through_unsynced_tables() -> None:
     actions = object()
 
@@ -722,6 +922,46 @@ async def test_wrapper_publishes_for_every_write_action(action_name: str) -> Non
     assert publish_calls == [("publish", "litellm_guardrailstable")]
 
 
+@pytest.mark.parametrize(
+    "action_name, expected_ids",
+    [
+        ("create", ("m-1",)),
+        ("update", ("m-1",)),
+        ("upsert", ("m-1",)),
+        ("delete", ()),
+        ("create_many", ()),
+        ("update_many", ()),
+        ("delete_many", ()),
+    ],
+)
+async def test_wrapper_publishes_the_written_model_id_for_row_writes(
+    action_name: str, expected_ids: Tuple[str, ...]
+) -> None:
+    publish_calls: List[Tuple[str, Tuple[str, ...]]] = []
+    wrapped = wrap_table_actions_for_config_sync(
+        actions=_ModelRowTableActions(),
+        table_name="litellm_proxymodeltable",
+        publish=_recording_publish_with_ids(publish_calls),
+    )
+
+    await getattr(wrapped, action_name)(data={})
+
+    assert publish_calls == [("litellm_proxymodeltable", expected_ids)]
+
+
+async def test_wrapper_publishes_no_model_ids_for_other_tables() -> None:
+    publish_calls: List[Tuple[str, Tuple[str, ...]]] = []
+    wrapped = wrap_table_actions_for_config_sync(
+        actions=_ModelRowTableActions(),
+        table_name="litellm_guardrailstable",
+        publish=_recording_publish_with_ids(publish_calls),
+    )
+
+    await wrapped.create(data={})
+
+    assert publish_calls == [("litellm_guardrailstable", ())]
+
+
 async def test_model_repository_write_publishes_via_live_coordination_cache() -> None:
     from litellm.proxy import proxy_server
     from litellm.proxy.proxy_server import _set_redis_usage_cache
@@ -747,7 +987,7 @@ async def test_model_repository_write_publishes_via_live_coordination_cache() ->
     assert len(client.published) == 1
     channel, message = client.published[0]
     assert channel == CONFIG_SYNC_CHANNEL
-    assert json.loads(message) == {"object_type": "litellm_proxymodeltable"}
+    assert json.loads(message) == {"object_type": "litellm_proxymodeltable", "model_ids": ["m-1"]}
 
 
 async def test_ui_settings_write_publishes_via_live_coordination_cache() -> None:
@@ -774,7 +1014,7 @@ async def test_ui_settings_write_publishes_via_live_coordination_cache() -> None
     assert len(client.published) == 1
     channel, message = client.published[0]
     assert channel == CONFIG_SYNC_CHANNEL
-    assert json.loads(message) == {"object_type": "litellm_uisettings"}
+    assert json.loads(message) == {"object_type": "litellm_uisettings", "model_ids": []}
 
 
 async def _publish_calls_for_invalidated_param(param_name: str) -> List[Tuple[str, str]]:
@@ -799,7 +1039,7 @@ async def test_invalidate_config_param_publishes_params_a_resync_applies(param_n
     assert len(published) == 1
     channel, message = published[0]
     assert channel == CONFIG_SYNC_CHANNEL
-    assert json.loads(message) == {"object_type": param_name}
+    assert json.loads(message) == {"object_type": param_name, "model_ids": []}
 
 
 @pytest.mark.parametrize("param_name", _STARTUP_ONLY_CONFIG_PARAM_NAMES)
@@ -932,6 +1172,46 @@ async def test_proxy_config_subscriber_resyncs_deployments_only() -> None:
     assert subscriber._task is None
 
 
+async def test_proxy_config_subscriber_applies_published_model_ids_from_the_db(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from litellm.proxy import proxy_server
+    from litellm.proxy.proxy_server import ProxyConfig
+
+    rows = [MagicMock()]
+    prisma_client = MagicMock()
+    prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=rows)
+    router = MagicMock()
+    router.registry.return_value.deployments.return_value = ["deployment"]
+    installed: List[object] = []
+
+    def install(db_models: object) -> None:
+        installed.append(db_models)
+
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "llm_model_list", None)
+    monkeypatch.setattr(proxy_server.proxy_config, "get_credentials", AsyncMock())
+    monkeypatch.setattr(proxy_server.proxy_config, "_add_deployment", install)
+    cache = _FakeRedisCache(_ScriptedPubSubRedisClient([_QueuePubSub()]))
+    config = ProxyConfig()
+
+    config.start_config_sync_subscriber(prisma_client=prisma_client, proxy_logging_obj=MagicMock(), redis_cache=cache)
+    subscriber = config.config_sync_subscriber
+    assert subscriber is not None
+    apply = subscriber._apply_model_ids
+    assert apply is not None
+    await apply(("m-1", "m-2"))
+    await config.stop_config_sync_subscriber()
+
+    prisma_client.db.litellm_proxymodeltable.find_many.assert_awaited_once_with(
+        where={"model_id": {"in": ["m-1", "m-2"]}}
+    )
+    assert installed == [rows]
+    assert proxy_server.llm_model_list == ["deployment"]
+
+
 async def test_proxy_config_does_not_start_subscriber_without_coordination_redis() -> None:
     from litellm.proxy.proxy_server import ProxyConfig
 

tests/unit/proxy/common_utils/test_registry_read_through.py

@@ -1,4 +1,5 @@
 import asyncio
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Final
 
 import pytest
@@ -6,7 +7,10 @@ import pytest
 from litellm.proxy.common_utils.registry_read_through import RegistryReadThrough
 
 if TYPE_CHECKING:
+    from litellm import Router
+    from litellm.proxy._types import LiteLLM_ProxyModelTable
     from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry
+    from litellm.types.utils import CredentialItem
 
 
 def nothing_loaded(_key: str) -> bool:
@@ -487,6 +491,178 @@ async def test_resync_model_deployments_loads_db_credentials_before_reconciling_
     installed.assert_called_once_with(db_models=rows, credential={"api_key": "sk-from-db"})
 
 
+@pytest.fixture
+def stored_params_salt(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("LITELLM_SALT_KEY", "sk-salt-stored-model-rows")
+
+
+def _stored_model_row(model_id: str, stored_params: Mapping[str, str]) -> "LiteLLM_ProxyModelTable":
+    from litellm.proxy._types import LiteLLM_ProxyModelTable
+    from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
+
+    return LiteLLM_ProxyModelTable(
+        model_id=model_id,
+        model_name="gpt-failover-group",
+        litellm_params={key: encrypt_value_helper(value) for key, value in stored_params.items()},
+        model_info={"id": model_id},
+    )
+
+
+def _backup_row(model_id: str, model: str = "openai/gpt-backup") -> "LiteLLM_ProxyModelTable":
+    return _stored_model_row(model_id, {"model": model, "api_key": "sk-backup"})
+
+
+def _router_serving(*model_ids: str) -> "Router":
+    from litellm import Router
+
+    return Router(
+        model_list=[
+            {
+                "model_name": "gpt-failover-group",
+                "litellm_params": {"model": "openai/gpt-primary", "api_key": "sk-primary"},
+                "model_info": {"id": model_id},
+            }
+            for model_id in model_ids
+        ]
+    )
+
+
+def _served_ids(deployments: "Sequence[Mapping[str, Mapping[str, object]]] | None") -> frozenset[object]:
+    return frozenset(deployment["model_info"]["id"] for deployment in deployments or ())
+
+
+class _ProxyModelTable:
+    def __init__(self, rows: "tuple[LiteLLM_ProxyModelTable, ...]") -> None:
+        self.rows = rows
+        self.reads = 0
+
+    async def find_many(
+        self, where: Mapping[str, Mapping[str, Sequence[str]]]
+    ) -> "tuple[LiteLLM_ProxyModelTable, ...]":
+        self.reads += 1
+        return tuple(row for row in self.rows if row.model_id in where["model_id"]["in"])
+
+    async def create(self, data: Mapping[str, object]) -> None:
+        raise AssertionError(f"a model load wrote a row: {data}")
+
+
+class _CredentialsTable:
+    def __init__(self, rows: "tuple[CredentialItem, ...]") -> None:
+        self.rows = rows
+
+    async def find_many(self) -> "tuple[CredentialItem, ...]":
+        return self.rows
+
+
+class _Db:
+    def __init__(
+        self, litellm_proxymodeltable: _ProxyModelTable, litellm_credentialstable: _CredentialsTable | None = None
+    ) -> None:
+        self.litellm_proxymodeltable = litellm_proxymodeltable
+        self.litellm_credentialstable = litellm_credentialstable or _CredentialsTable(())
+        self.other_tables_read: list[str] = []
+
+    def __getattr__(self, table_name: str) -> object:
+        self.other_tables_read.append(table_name)
+        raise AttributeError(table_name)
+
+
+class _PrismaClientWithoutConfigRows:
+    def __init__(self, db: object) -> None:
+        self.db = db
+
+    async def get_generic_data(self, key: str, value: str, table_name: str) -> None:
+        return None
+
+
+def _serve_db_models(monkeypatch: pytest.MonkeyPatch, db: object, router: "Router | None") -> None:
+    import litellm.proxy.proxy_server as proxy_server
+
+    monkeypatch.setattr(proxy_server, "prisma_client", _PrismaClientWithoutConfigRows(db))
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "llm_model_list", None)
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("stored_params_salt")
+async def test_resync_model_deployments_by_ids_installs_only_the_published_rows(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
+
+    router: Final = _router_serving("primary-id")
+    _serve_db_models(
+        monkeypatch, _Db(_ProxyModelTable((_backup_row("m-1"), _backup_row("m-2"), _backup_row("m-3")))), router
+    )
+
+    await resync_model_deployments_by_ids(("m-1", "m-2"))
+
+    assert _served_ids(router.registry().deployments()) == {"primary-id", "m-1", "m-2"}
+    assert _served_ids(proxy_server.llm_model_list) == {"primary-id", "m-1", "m-2"}
+    assert not proxy_server.MODEL_RECONCILE_LOCK.locked()
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("stored_params_salt")
+async def test_resync_model_deployments_by_ids_leaves_the_router_alone_for_deleted_rows(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
+
+    router: Final = _router_serving("primary-id")
+    _serve_db_models(monkeypatch, _Db(_ProxyModelTable(())), router)
+
+    await resync_model_deployments_by_ids(("deleted-id",))
+
+    assert _served_ids(router.registry().deployments()) == {"primary-id"}
+    assert proxy_server.llm_model_list is None
+    assert not proxy_server.MODEL_RECONCILE_LOCK.locked()
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("stored_params_salt")
+async def test_resync_model_deployments_by_ids_without_a_router_leaves_the_reload_to_the_debounced_resync(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
+
+    table: Final = _ProxyModelTable((_backup_row("m-1"),))
+    db: Final = _Db(table)
+    _serve_db_models(monkeypatch, db, None)
+
+    await resync_model_deployments_by_ids(("m-1",))
+
+    assert proxy_server.llm_router is None
+    assert table.reads == 1
+    assert db.other_tables_read == []
+
+
+@pytest.mark.asyncio
+async def test_resync_model_deployments_by_ids_respects_supported_db_objects(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
+
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(
+        side_effect=AssertionError("db hit for an object type this replica does not load")
+    )
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["guardrails"]})
+
+    await resync_model_deployments_by_ids(("gated-out-id",))
+
+    prisma_client.db.litellm_proxymodeltable.find_many.assert_not_awaited()
+
+
 @pytest.mark.asyncio
 async def test_resync_model_deployments_respects_supported_db_objects(monkeypatch):
     from unittest.mock import AsyncMock, MagicMock
@@ -786,3 +962,174 @@ async def test_resync_model_deployments_reconciles_every_db_row_when_the_missed_
 
     assert await _resync_model_deployments(router_name) is router_holds_the_name_after_reconcile
     reconciled.assert_called_once_with(prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj)
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("stored_params_salt")
+async def test_resync_model_deployments_by_ids_reads_the_writer_not_a_lagging_replica(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from unittest.mock import MagicMock
+
+    from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
+    from litellm.proxy.db.prisma_client import PrismaWrapper
+    from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
+
+    writer: Final = MagicMock(name="writer_prisma")
+    writer.litellm_proxymodeltable = _ProxyModelTable((_backup_row("backup-id"),))
+    replica_table: Final = _ProxyModelTable(())
+    replica: Final = MagicMock(name="lagging_replica_prisma")
+    replica.litellm_proxymodeltable = replica_table
+    router: Final = _router_serving("primary-id")
+    _serve_db_models(
+        monkeypatch,
+        RoutingPrismaWrapper(
+            writer=PrismaWrapper(original_prisma=writer, iam_token_db_auth=False),
+            reader=PrismaWrapper(original_prisma=replica, iam_token_db_auth=False),
+        ),
+        router,
+    )
+
+    await resync_model_deployments_by_ids(("backup-id",))
+
+    assert router.registry().has_deployment("backup-id")
+    assert replica_table.reads == 0
+
+
+@pytest.mark.asyncio
+async def test_unknown_model_name_read_through_reads_the_replica_without_waiting_for_a_reload(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from types import SimpleNamespace
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments
+    from litellm.proxy.db.prisma_client import PrismaWrapper
+    from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
+
+    writer: Final = MagicMock(name="writer_prisma")
+    writer.litellm_proxymodeltable = SimpleNamespace(find_many=AsyncMock(return_value=[]), create=AsyncMock())
+    replica: Final = MagicMock(name="replica_prisma")
+    replica.litellm_proxymodeltable = SimpleNamespace(find_many=AsyncMock(return_value=[]), create=AsyncMock())
+    prisma_client: Final = MagicMock()
+    prisma_client.db = RoutingPrismaWrapper(
+        writer=PrismaWrapper(original_prisma=writer, iam_token_db_auth=False),
+        reader=PrismaWrapper(original_prisma=replica, iam_token_db_auth=False),
+    )
+    reload_in_progress: Final = asyncio.Lock()
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "llm_router", MagicMock())
+    monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", reload_in_progress)
+
+    async with reload_in_progress:
+        found: Final = await asyncio.wait_for(_resync_model_deployments("no-such-model"), timeout=2)
+
+    assert found is False
+    writer.litellm_proxymodeltable.find_many.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("stored_params_salt")
+async def test_resync_model_deployments_by_ids_installs_the_row_a_concurrent_reload_left_current(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
+
+    table: Final = _ProxyModelTable((_backup_row("m-1", model="openai/gpt-before-update"),))
+    router: Final = _router_serving("primary-id")
+    _serve_db_models(monkeypatch, _Db(table), router)
+    reload_in_progress: Final = asyncio.Lock()
+    monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", reload_in_progress)
+
+    async with reload_in_progress:
+        targeted_load: Final = asyncio.create_task(resync_model_deployments_by_ids(("m-1",)))
+        for _ in range(5):
+            await asyncio.sleep(0)
+        assert not router.registry().has_deployment("m-1")
+        table.rows = (_backup_row("m-1", model="openai/gpt-after-update"),)
+    await targeted_load
+
+    installed: Final = router.registry().deployment("m-1")
+    assert installed is not None
+    assert installed.litellm_params.model == "openai/gpt-after-update"
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("stored_params_salt")
+@pytest.mark.parametrize(
+    ("stored_credential_name", "credentials_on_this_worker", "api_key_this_worker_uses"),
+    [
+        ("shared-openai", (), "sk-from-db"),
+        ("shared-openai", ("another-openai",), "sk-from-db"),
+        ("shared-openai", ("shared-openai",), "sk-on-this-worker"),
+        (None, (), None),
+    ],
+)
+async def test_resync_model_deployments_by_ids_reloads_credentials_only_for_one_this_worker_lacks(
+    monkeypatch: pytest.MonkeyPatch,
+    stored_credential_name: str | None,
+    credentials_on_this_worker: tuple[str, ...],
+    api_key_this_worker_uses: str | None,
+) -> None:
+    import litellm
+    from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+    from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
+    from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
+    from litellm.types.utils import CredentialItem
+
+    model_params: Final = {"model": "openai/gpt-backup"}
+    row: Final = _stored_model_row(
+        "backup-id",
+        model_params if stored_credential_name is None else model_params | {"litellm_credential_name": stored_credential_name},
+    )
+    stored_credential: Final = CredentialItem(
+        credential_name="shared-openai",
+        credential_values={"api_key": encrypt_value_helper("sk-from-db")},
+        credential_info={},
+    )
+    router: Final = _router_serving("primary-id")
+    monkeypatch.setattr(
+        litellm,
+        "credential_list",
+        [
+            CredentialItem(credential_name=name, credential_values={"api_key": "sk-on-this-worker"}, credential_info={})
+            for name in credentials_on_this_worker
+        ],
+    )
+    _serve_db_models(monkeypatch, _Db(_ProxyModelTable((row,)), _CredentialsTable((stored_credential,))), router)
+
+    await resync_model_deployments_by_ids(("backup-id",))
+
+    credential: Final = CredentialAccessor.find_credential("shared-openai")
+    assert (credential.credential_values.get("api_key") if credential else None) == api_key_this_worker_uses
+    assert router.registry().has_deployment("backup-id")
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("stored_params_salt")
+async def test_resync_model_deployments_by_ids_installs_the_ordinary_rows_queued_with_an_auto_router(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
+
+    table: Final = _ProxyModelTable(
+        (
+            _stored_model_row("auto-router-id", {"model": "auto_router/complexity-router"}),
+            _backup_row("backup-id"),
+        )
+    )
+    db: Final = _Db(table)
+    router: Final = _router_serving("primary-id")
+    _serve_db_models(monkeypatch, db, router)
+
+    await resync_model_deployments_by_ids(("auto-router-id", "backup-id"))
+
+    assert _served_ids(router.registry().deployments()) == {"primary-id", "backup-id"}
+    assert _served_ids(proxy_server.llm_model_list) == {"primary-id", "backup-id"}
+    assert table.reads == 1
+    assert db.other_tables_read == []
+    assert not proxy_server.MODEL_RECONCILE_LOCK.locked()

tests/unit/proxy/db/test_db_spend_update_writer.py

@@ -6,6 +6,7 @@ import re
 from collections.abc import AsyncIterator, Callable
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
 from datetime import datetime, timedelta, timezone
+from pathlib import Path
 from types import SimpleNamespace
 from typing import Final, cast
 from unittest.mock import AsyncMock, MagicMock, call, patch
@@ -4325,6 +4326,36 @@ async def test_insert_spend_log_asks_for_an_immediate_flush_only_on_rows_the_bri
     assert prisma.spend_log_flush_requested.is_set() is expects_flush
 
 
+
+@pytest.mark.asyncio
+@pytest.mark.timeout(30)
+async def test_a_chatgpt_responses_row_queues_without_a_default_chatgpt_login(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+):
+    monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "no-default-login"))
+    monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False)
+    own_login: Final = tmp_path / "team" / "auth.json"
+    own_login.parent.mkdir()
+    own_login.write_text(json.dumps({"access_token": "at-team", "account_id": "acct-team"}))
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": "chatgpt-team",
+                "litellm_params": {"model": "chatgpt/gpt-5.6-terra", "chatgpt_auth_file": str(own_login)},
+                "model_info": {"id": "dep-chatgpt"},
+            }
+        ]
+    )
+    prisma: Final = _tool_usage_prisma()
+    prisma.spend_log_flush_requested = asyncio.Event()
+    payload: Final = {"request_id": "req-1", "call_type": "aresponses", "model_id": "dep-chatgpt"}
+
+    await DBSpendUpdateWriter()._insert_spend_log_to_db(payload=payload, prisma_client=prisma, llm_router=router)
+
+    assert prisma.spend_log_transactions == [payload]
+    assert prisma.spend_log_flush_requested.is_set() is False
+
+
 def _batch_cost_payload() -> dict:
     return {
         **_minimal_spend_payload(),

tests/unit/proxy/management_endpoints/scim/test_scim_patch_user.py

@@ -502,9 +502,9 @@ def test_apply_patch_ops_remove_group_filtered_path_without_value():
         Operations=[SCIMPatchOperation(op="remove", path='groups[value eq "team-1"]')]
     )
 
-    _, final_team_set = _apply_patch_ops(existing_user=user, patch_ops=patch_ops)
+    _, final_teams = _apply_patch_ops(existing_user=user, patch_ops=patch_ops)
 
-    assert final_team_set == {"team-2"}
+    assert final_teams == ("team-2",)
 
 
 def test_apply_patch_ops_add_group_filtered_path_without_value():
@@ -519,9 +519,9 @@ def test_apply_patch_ops_add_group_filtered_path_without_value():
         Operations=[SCIMPatchOperation(op="add", path="groups[value eq 'team-3']")]
     )
 
-    _, final_team_set = _apply_patch_ops(existing_user=user, patch_ops=patch_ops)
+    _, final_teams = _apply_patch_ops(existing_user=user, patch_ops=patch_ops)
 
-    assert final_team_set == {"team-1", "team-3"}
+    assert final_teams == ("team-1", "team-3")
 
 
 def test_apply_patch_ops_replace_groups_empty_value_does_not_use_path_filter():
@@ -539,9 +539,70 @@ def test_apply_patch_ops_replace_groups_empty_value_does_not_use_path_filter():
         ]
     )
 
-    _, final_team_set = _apply_patch_ops(existing_user=user, patch_ops=patch_ops)
+    _, final_teams = _apply_patch_ops(existing_user=user, patch_ops=patch_ops)
 
-    assert final_team_set == set()
+    assert final_teams == ()
+
+
+@pytest.mark.parametrize(
+    "operations,expected_teams",
+    [
+        pytest.param(
+            [
+                SCIMPatchOperation(op="add", path="groups", value=[{"value": "team-mu"}, {"value": "team-beta"}]),
+                SCIMPatchOperation(op="remove", path='groups[value eq "team-kappa"]'),
+                SCIMPatchOperation(op="add", path="groups", value=[{"value": "team-alpha"}, {"value": "team-omega"}]),
+            ],
+            ("team-zeta", "team-alpha", "team-delta", "team-mu", "team-beta", "team-omega"),
+            id="adds_follow_the_record_in_op_order",
+        ),
+        pytest.param(
+            [
+                SCIMPatchOperation(
+                    op="replace",
+                    path="groups",
+                    value=[{"value": "team-omega"}, {"value": "team-delta"}, {"value": "team-beta"}],
+                )
+            ],
+            ("team-delta", "team-omega", "team-beta"),
+            id="replace_keeps_teams_the_user_has_in_record_order_and_adds_new_ones_in_request_order",
+        ),
+        pytest.param(
+            [
+                SCIMPatchOperation(
+                    op="replace",
+                    path="groups",
+                    value=[{"value": "team-omega"}, {"value": "team-delta"}, {"value": "team-beta"}],
+                ),
+                SCIMPatchOperation(op="add", path="groups", value=[{"value": "team-mu"}]),
+                SCIMPatchOperation(op="remove", path='groups[value eq "team-delta"]'),
+            ],
+            ("team-delta", "team-omega", "team-beta"),
+            id="a_replace_wins_over_later_group_ops",
+        ),
+        pytest.param(
+            [
+                SCIMPatchOperation(op="remove", path='groups[value eq "team-zeta"]'),
+                SCIMPatchOperation(op="add", path="groups", value=[{"value": "team-zeta"}]),
+            ],
+            ("team-zeta", "team-alpha", "team-kappa", "team-delta"),
+            id="removing_and_re_adding_a_team_keeps_its_place",
+        ),
+    ],
+)
+def test_apply_patch_ops_keeps_team_order_so_the_first_team_stays_first(
+    operations: list[SCIMPatchOperation], expected_teams: tuple[str, ...]
+):
+    user: Final = LiteLLM_UserTable(
+        user_id="user-order",
+        user_email="order@example.com",
+        teams=["team-zeta", "team-alpha", "team-kappa", "team-delta"],
+        metadata={},
+    )
+
+    _, final_teams = _apply_patch_ops(existing_user=user, patch_ops=SCIMPatchOp(Operations=operations))
+
+    assert final_teams == expected_teams
 
 
 @pytest.mark.asyncio

tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py

@@ -25,6 +25,7 @@ from litellm.proxy._types import (
     NewUserResponse,
     ProxyErrorTypes,
     ProxyException,
+    TeamMemberAddRequest,
     UserAPIKeyAuth,
 )
 from litellm.proxy.management_endpoints.scim.scim_v2 import (
@@ -941,6 +942,57 @@ async def test_handle_existing_user_by_email_syncs_roster_and_dedups_teams(mocke
     assert update_calls[0].kwargs["data"]["teams"] == ["team-a", "team-b"]
 
 
+class _EmailMatchedUserRow:
+    def __init__(self, user: LiteLLM_UserTable) -> None:
+        self.user = user
+
+    async def find_first(self, *, where: dict[str, object]) -> LiteLLM_UserTable:
+        return self.user
+
+    async def update(self, *, where: dict[str, object], data: dict[str, object]) -> LiteLLM_UserTable:
+        self.user = self.user.model_copy(update=data)
+        return self.user
+
+
+@pytest.mark.asyncio
+async def test_handle_existing_user_by_email_keeps_record_team_order(mocker: MockerFixture):
+    user_rows: Final = _EmailMatchedUserRow(
+        LiteLLM_UserTable(
+            user_id="same-id",
+            user_email="member@example.com",
+            user_alias="Member",
+            teams=["team-zeta", "team-alpha", "team-kappa"],
+            metadata={},
+        )
+    )
+    mock_prisma_client: Final = mocker.MagicMock()
+    mock_prisma_client.db = mocker.MagicMock()
+    mock_prisma_client.db.litellm_usertable = user_rows
+
+    mocker.patch(  # test-quality-ok: roster helpers are module-level, not injectable into the helper
+        "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user",
+        AsyncMock(return_value=None),
+    )
+    mocker.patch(  # test-quality-ok: roster helpers are module-level, not injectable into the helper
+        "litellm.proxy.management_endpoints.scim.scim_v2._handle_team_membership_changes",
+        AsyncMock(),
+    )
+
+    await UserProvisionerHelpers.handle_existing_user_by_email(
+        prisma_client=mock_prisma_client,
+        new_user_request=NewUserRequest(
+            user_id="same-id",
+            user_email="member@example.com",
+            user_alias="Member",
+            teams=["team-mu", "team-kappa", "team-beta", "team-zeta"],
+            metadata={},
+            auto_create_key=False,
+        ),
+    )
+
+    assert user_rows.user.teams == ["team-zeta", "team-kappa", "team-mu", "team-beta"]
+
+
 @pytest.mark.asyncio
 async def test_handle_existing_user_by_email_without_teams_preserves_memberships(mocker):
     """Adoption via POST /Users without ``groups`` must keep the user's existing teams.
@@ -1290,6 +1342,49 @@ async def test_handle_team_membership_changes_add_and_remove(mocker):
     assert call_args[1]["teams_ids_to_remove_user_from"] == ["team1"]
 
 
+class _TeamRosterRecord:
+    def __init__(self, missing_team: str) -> None:
+        self.missing_team = missing_team
+        self.teams: tuple[str, ...] = ()
+
+    async def team_member_add(self, *, data: TeamMemberAddRequest, user_api_key_dict: UserAPIKeyAuth) -> None:
+        if data.team_id == self.missing_team:
+            raise HTTPException(status_code=404, detail={"error": "Team not found"})
+        self.teams = (*self.teams, data.team_id)
+
+
+@pytest.mark.asyncio
+async def test_handle_team_membership_changes_adds_teams_in_request_order_when_one_fails(mocker: MockerFixture):
+    requested_teams: Final = (
+        "team-zeta",
+        "team-alpha",
+        "team-kappa",
+        "team-kept",
+        "team-mu",
+        "team-missing",
+        "team-beta",
+        "team-omega",
+        "team-delta",
+        "team-sigma",
+        "team-gamma",
+        "team-tau",
+    )
+    roster: Final = _TeamRosterRecord(missing_team="team-missing")
+    mocker.patch(  # test-quality-ok: team_member_add is module-level, not injectable into the helper
+        "litellm.proxy.management_endpoints.scim.scim_v2.team_member_add",
+        roster.team_member_add,
+    )
+
+    with pytest.raises(SCIMRosterSyncError):
+        await _handle_team_membership_changes(
+            user_id="test-user",
+            existing_teams=["team-kept"],
+            new_teams=list(requested_teams),
+        )
+
+    assert roster.teams == tuple(team for team in requested_teams if team not in {"team-kept", "team-missing"})
+
+
 @pytest.mark.asyncio
 async def test_update_user_success(mocker):
     """Should successfully update user with PUT request"""
@@ -1492,6 +1587,57 @@ async def test_update_user_without_groups_preserves_memberships_and_role(mocker,
     assert "user_role" not in update_data
 
 
+@pytest.mark.asyncio
+async def test_update_user_put_keeps_the_record_order_of_teams_the_user_already_has(
+    mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from litellm.proxy.proxy_server import proxy_config
+
+    async def no_scim_admin_group() -> dict[str, object]:
+        return {}
+
+    monkeypatch.setattr(proxy_config, "get_config", no_scim_admin_group)
+    monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False)
+
+    existing_user: Final = mocker.MagicMock()
+    existing_user.teams = ["team-zeta", "team-alpha", "team-kappa"]
+    existing_user.metadata = {}
+    scim_user: Final = SCIMUser(
+        schemas=["urn:ietf:params:scim:schemas:core:2.0:User"],
+        userName="order-user",
+        emails=[SCIMUserEmail(value="order@example.com")],
+        groups=[
+            SCIMUserGroup(value="team-mu"),
+            SCIMUserGroup(value="team-kappa"),
+            SCIMUserGroup(value="team-beta"),
+            SCIMUserGroup(value="team-zeta"),
+        ],
+    )
+    mock_prisma_client: Final = mocker.MagicMock()
+    mock_prisma_client.db.litellm_usertable.update = AsyncMock(return_value={"user_id": "order-user"})
+    mocker.patch(  # test-quality-ok: update_user's collaborators are module-level, not injectable
+        "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception",
+        AsyncMock(return_value=mock_prisma_client),
+    )
+    mocker.patch(  # test-quality-ok: update_user's collaborators are module-level, not injectable
+        "litellm.proxy.management_endpoints.scim.scim_v2._check_user_exists",
+        AsyncMock(return_value=existing_user),
+    )
+    mocker.patch(  # test-quality-ok: update_user's collaborators are module-level, not injectable
+        "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user",
+        AsyncMock(return_value=scim_user),
+    )
+    mocker.patch(  # test-quality-ok: roster writes are module-level, not injectable
+        "litellm.proxy.management_endpoints.scim.scim_v2.patch_team_membership",
+        AsyncMock(),
+    )
+
+    await update_user(user_id="order-user", user=scim_user)
+
+    update_data: Final = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs["data"]
+    assert update_data["teams"] == ["team-zeta", "team-kappa", "team-mu", "team-beta"]
+
+
 @pytest.mark.asyncio
 async def test_update_user_not_found(mocker):
     """Should raise 404 when user doesn't exist"""

tests/unit/proxy/management_endpoints/test_access_group_management.py

@@ -687,33 +687,152 @@ class _FakeAccessGroupBudgetTable:
         return self.rows.pop(access_group_name, None)
 
 
+class _FakeClock:
+    def __init__(self) -> None:
+        self.now = 0.0
+        self.slept: list[float] = []
+
+    def __call__(self) -> float:
+        return self.now
+
+    async def sleep(self, seconds: float) -> None:
+        self.slept.append(seconds)
+        self.now += seconds
+
+
 class _FakeModelTable:
-    def __init__(self, journal: list[str], deployments) -> None:
+    def __init__(self, journal: list[str], deployments, failing_model_ids=frozenset()) -> None:
         self.journal = journal
         self.deployments = list(deployments)
+        self.failing_model_ids = failing_model_ids
         self.updates: list[tuple[dict, dict]] = []
+        self.rows_locked_by_another_session: frozenset[str] = frozenset()
+        self.group_locked_by_another_session = False
+        self.edits_the_other_session_commits: dict[str, dict] = {}
+        self.other_session_commits_after_refusals: int | None = None
+        self.lock_refusals = 0
 
     async def find_many(self, where=None, **kwargs):
-        return list(self.deployments)
+        model_names = (where or {}).get("model_name", {}).get("in")
+        return [d for d in self.deployments if model_names is None or d.model_name in model_names]
+
+    def lock_group(self, wait):
+        if not self.group_locked_by_another_session:
+            return True
+        if wait:
+            raise TimeoutError("lock_timeout expired waiting for the access group lock")
+        self._refuse_lock()
+        return False
+
+    def lock_rows(self, model_ids, skip_locked):
+        held = self.rows_locked_by_another_session.intersection(model_ids)
+        if not held:
+            return list(model_ids)
+        if not skip_locked:
+            raise TimeoutError(f"timed out waiting for the row lock on {sorted(held)}")
+        self._refuse_lock()
+        return [model_id for model_id in model_ids if model_id not in held]
+
+    def _refuse_lock(self):
+        self.lock_refusals += 1
+        if self.lock_refusals == self.other_session_commits_after_refusals:
+            self._other_session_commits()
+
+    def _other_session_commits(self):
+        self.deployments = [
+            _FakeDeployment(
+                model_id=d.model_id,
+                model_name=d.model_name,
+                model_info=self.edits_the_other_session_commits.get(d.model_id, d.model_info),
+            )
+            for d in self.deployments
+        ]
+        self.rows_locked_by_another_session = frozenset()
+        self.group_locked_by_another_session = False
 
     async def find_unique(self, where, include=None):
         return next((d for d in self.deployments if d.model_id == where["model_id"]), None)
 
     async def update(self, where, data, include=None):
+        if where["model_id"] in self.failing_model_ids:
+            raise RuntimeError(f"database write to {where['model_id']} failed")
         self.journal.append(f"model_table.update:{where['model_id']}")
         self.updates.append((dict(where), dict(data)))
         return None
 
+    async def create(self, data, **kwargs):
+        raise AssertionError("access group endpoints never create deployments")
+
+
+class _FakeModelTableBatch:
+    """Queues model-table updates and applies them all or none when the batch closes, like a Prisma batch."""
+
+    def __init__(self, model_table: _FakeModelTable) -> None:
+        self.model_table = model_table
+        self.queued: list[tuple[dict, dict]] = []
+        self.litellm_proxymodeltable = SimpleNamespace(update=self._queue_update)
+
+    def _queue_update(self, where, data):
+        self.queued.append((dict(where), dict(data)))
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        if exc_type is not None:
+            return False
+        failing = [where["model_id"] for where, _ in self.queued if where["model_id"] in self.model_table.failing_model_ids]
+        if failing:
+            raise RuntimeError(f"batch write to {failing} failed, nothing applied")
+        for where, data in self.queued:
+            await self.model_table.update(where=where, data=data)
+        return False
+
+
+class _FakeModelTableTx:
+    """An interactive transaction on the primary that journals the row lock and the read it guards."""
+
+    def __init__(self, journal: list[str], model_table: _FakeModelTable) -> None:
+        self.journal = journal
+        self.model_table = model_table
+        self.litellm_proxymodeltable = SimpleNamespace(find_many=self._find_many)
+
+    async def query_raw(self, query, *args):
+        if "advisory_xact_lock" in query:
+            self.journal.append(f"access_group.lock:{args[0]}")
+            return [{"locked": self.model_table.lock_group(wait="pg_try_" not in query)}]
+        if "FOR UPDATE" in query:
+            model_ids = sorted(args[0] if args else (d.model_id for d in self.model_table.deployments))
+            self.journal.append(f"model_table.lock:{','.join(model_ids)}")
+            locked = self.model_table.lock_rows(model_ids, skip_locked="SKIP LOCKED" in query)
+            return [{"model_id": model_id} for model_id in locked]
+        self.journal.append(f"model_table.query:{query}")
+        return []
+
+    async def _find_many(self):
+        self.journal.append("model_table.read")
+        return await self.model_table.find_many()
+
+    def batch_(self):
+        return _FakeModelTableBatch(self.model_table)
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        return False
+
 
 class _FakePrismaClient:
-    def __init__(self, journal: list[str], deployments=()) -> None:
+    def __init__(self, journal: list[str], deployments=(), failing_model_ids=frozenset()) -> None:
         self.budget_table = _FakeBudgetTable(journal)
         self.access_group_budget_table = _FakeAccessGroupBudgetTable(journal, self.budget_table)
-        self.model_table = _FakeModelTable(journal, deployments)
+        self.model_table = _FakeModelTable(journal, deployments, failing_model_ids)
         self.db = SimpleNamespace(
             litellm_budgettable=self.budget_table,
             litellm_modelaccessgroupbudgettable=self.access_group_budget_table,
             litellm_proxymodeltable=self.model_table,
+            tx=lambda timeout=None: _FakeModelTableTx(journal, self.model_table),
         )
 
     def jsonify_object(self, data):
@@ -784,6 +903,12 @@ def _proxy_with_stubbed_reload(prisma):
         yield
 
 
+def _serve_model_groups(router, model_groups):
+    router.registry.return_value.names.side_effect = lambda model_name=None, *, kind="groups", **_: (
+        list(model_groups) if kind == "groups" else []
+    )
+
+
 def _eviction_journal(access_group):
     """Both auth cache keys, in the order a write path has to evict them."""
     from litellm.proxy.common_utils.user_api_key_cache import (
@@ -1290,3 +1415,605 @@ async def test_deleting_an_access_group_that_never_had_a_budget_still_evicts():
     assert response.models_updated == 1
     assert prisma.access_group_budget_table.rows == {}
     _assert_evicted_after_write(journal, "prod-models", "access_group_budget.delete:prod-models")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "selection",
+    [{"model_ids": ["stays", "joins"]}, {"model_names": ["gpt-stays", "gpt-joins"]}],
+)
+async def test_updating_an_access_group_writes_only_the_deployments_that_join_or_leave_it(selection):
+    import json
+
+    import litellm.proxy.proxy_server as proxy_server
+
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient(
+        [],
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="leaves", model_name="gpt-leaves"),
+            _deployment(model_id="joins", model_name="gpt-joins", access_groups=()),
+        ],
+    )
+
+    with _proxy_with_stubbed_reload(prisma):
+        _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-leaves", "gpt-joins"])
+        response = await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(**selection),
+            user_api_key_dict=_admin(),
+        )
+
+    written_access_groups = {
+        where["model_id"]: json.loads(data["model_info"])["access_groups"]
+        for where, data in prisma.model_table.updates
+    }
+    assert written_access_groups == {"joins": ["prod-models"], "leaves": []}
+    assert response.models_updated == 2
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "selection",
+    [{"model_ids": ["stays", "joins"]}, {"model_names": ["gpt-stays", "gpt-joins"]}],
+)
+async def test_updating_an_access_group_reads_the_primary_not_a_lagging_replica(selection):
+    import json
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.db.prisma_client import PrismaWrapper
+    from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    primary = _FakePrismaClient(
+        [],
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="leaves", model_name="gpt-leaves"),
+            _deployment(model_id="joins", model_name="gpt-joins", access_groups=()),
+        ],
+    )
+    replica_before_the_last_edit = _FakePrismaClient(
+        [],
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="leaves", model_name="gpt-leaves", access_groups=()),
+            _deployment(model_id="joins", model_name="gpt-joins"),
+        ],
+    )
+    primary.db = RoutingPrismaWrapper(
+        writer=PrismaWrapper(original_prisma=primary.db, iam_token_db_auth=False),
+        reader=PrismaWrapper(original_prisma=replica_before_the_last_edit.db, iam_token_db_auth=False),
+    )
+
+    with _proxy_with_stubbed_reload(primary):
+        _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-leaves", "gpt-joins"])
+        await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(**selection),
+            user_api_key_dict=_admin(),
+        )
+
+    written_access_groups = {
+        where["model_id"]: json.loads(data["model_info"])["access_groups"]
+        for where, data in primary.model_table.updates
+    }
+    assert written_access_groups == {"joins": ["prod-models"], "leaves": []}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("failing_model_id", ["leaves", "joins"])
+async def test_a_failed_access_group_update_leaves_every_deployment_unchanged(failing_model_id):
+    from fastapi import HTTPException
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    deployments = [
+        _deployment(model_id="stays", model_name="gpt-stays"),
+        _deployment(model_id="leaves", model_name="gpt-leaves"),
+        _deployment(model_id="joins", model_name="gpt-joins", access_groups=()),
+    ]
+    prisma = _FakePrismaClient([], deployments=deployments, failing_model_ids=frozenset({failing_model_id}))
+
+    with _proxy_with_stubbed_reload(prisma):
+        _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-leaves", "gpt-joins"])
+        with pytest.raises(HTTPException) as raised:
+            await update_access_group(
+                access_group="prod-models",
+                data=UpdateModelGroupRequest(model_ids=["stays", "joins"]),
+                user_api_key_dict=_admin(),
+            )
+
+    assert raised.value.status_code == 500
+    assert prisma.model_table.updates == []
+
+
+@pytest.mark.asyncio
+async def test_an_access_group_update_keeps_a_model_info_field_another_admin_set_while_names_were_validated():
+    import json
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient(
+        [],
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="joins", model_name="gpt-joins", access_groups=()),
+        ],
+    )
+
+    def another_admin_edits_joins_while_names_are_checked(model_name=None, *, kind="groups", **_):
+        if kind != "groups":
+            return []
+        prisma.model_table.deployments = [
+            deployment
+            if deployment.model_id != "joins"
+            else _FakeDeployment(
+                model_id="joins",
+                model_name="gpt-joins",
+                model_info={"access_groups": [], "description": "set by another admin"},
+            )
+            for deployment in prisma.model_table.deployments
+        ]
+        return ["gpt-stays", "gpt-joins"]
+
+    with _proxy_with_stubbed_reload(prisma):
+        proxy_server.llm_router.registry.return_value.names.side_effect = (
+            another_admin_edits_joins_while_names_are_checked
+        )
+        await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(model_names=["gpt-stays", "gpt-joins"]),
+            user_api_key_dict=_admin(),
+        )
+
+    written = {where["model_id"]: json.loads(data["model_info"]) for where, data in prisma.model_table.updates}
+    assert written["joins"]["description"] == "set by another admin"
+    assert written["joins"]["access_groups"] == ["prod-models"]
+
+
+@pytest.mark.asyncio
+async def test_an_access_group_update_locks_the_rows_it_reads_and_tells_other_workers_after_the_commit():
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    journal: list[str] = []
+    prisma = _FakePrismaClient(
+        journal,
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="leaves", model_name="gpt-leaves"),
+            _deployment(model_id="joins", model_name="gpt-joins", access_groups=()),
+        ],
+    )
+
+    async def record_publish(model_ids):
+        journal.append(f"publish:{','.join(sorted(model_ids))}")
+
+    with (
+        _proxy_with_stubbed_reload(prisma),
+        patch(  # test-quality-ok: the endpoint publishes through its module-level import; Redis is the real seam
+            "litellm.proxy.management_endpoints.model_access_group_management_endpoints.publish_model_rows_written",
+            new=record_publish,
+        ),
+    ):
+        await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(model_ids=["stays", "joins"]),
+            user_api_key_dict=_admin(),
+        )
+
+    assert journal == [
+        "access_group.lock:prod-models",
+        "model_table.read",
+        "model_table.lock:joins,leaves,stays",
+        "model_table.read",
+        "model_table.update:leaves",
+        "model_table.update:joins",
+        "publish:joins,leaves",
+    ]
+
+
+@pytest.mark.asyncio
+async def test_an_access_group_update_that_changes_no_membership_writes_and_announces_nothing():
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    journal: list[str] = []
+    prisma = _FakePrismaClient(
+        journal,
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="also-stays", model_name="gpt-also-stays"),
+            _deployment(model_id="outside", model_name="gpt-outside", access_groups=()),
+        ],
+    )
+
+    async def record_publish(model_ids):
+        journal.append(f"publish:{','.join(sorted(model_ids))}")
+
+    with (
+        _proxy_with_stubbed_reload(prisma),
+        patch(  # test-quality-ok: the endpoint publishes through its module-level import; Redis is the real seam
+            "litellm.proxy.management_endpoints.model_access_group_management_endpoints.publish_model_rows_written",
+            new=record_publish,
+        ),
+    ):
+        response = await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(model_ids=["stays", "also-stays"]),
+            user_api_key_dict=_admin(),
+        )
+
+    assert response.models_updated == 2
+    assert journal == [
+        "access_group.lock:prod-models",
+        "model_table.read",
+        "model_table.lock:also-stays,stays",
+        "model_table.read",
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("ignored_model_names", [None, ["gpt-unrelated"]])
+async def test_an_access_group_update_goes_through_while_another_session_holds_a_model_row_outside_the_group(
+    ignored_model_names,
+):
+    import json
+
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient(
+        [],
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="leaves", model_name="gpt-leaves"),
+            _deployment(model_id="joins", model_name="gpt-joins", access_groups=()),
+            _deployment(model_id="unrelated", model_name="gpt-unrelated", access_groups=("other-group",)),
+        ],
+    )
+    prisma.model_table.rows_locked_by_another_session = frozenset({"unrelated"})
+
+    with _proxy_with_stubbed_reload(prisma):
+        response = await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(model_ids=["stays", "joins"], model_names=ignored_model_names),
+            user_api_key_dict=_admin(),
+        )
+
+    written_access_groups = {
+        where["model_id"]: json.loads(data["model_info"])["access_groups"]
+        for where, data in prisma.model_table.updates
+    }
+    assert written_access_groups == {"joins": ["prod-models"], "leaves": []}
+    assert response.models_updated == 2
+
+
+@pytest.mark.asyncio
+async def test_an_access_group_update_retries_a_locked_row_and_keeps_the_edit_committed_there():
+    import json
+
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient(
+        [],
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="joins", model_name="gpt-joins", access_groups=()),
+        ],
+    )
+    prisma.model_table.rows_locked_by_another_session = frozenset({"joins"})
+    prisma.model_table.edits_the_other_session_commits = {
+        "joins": {"access_groups": [], "description": "saved while the update waited"}
+    }
+    prisma.model_table.other_session_commits_after_refusals = 1
+
+    with _proxy_with_stubbed_reload(prisma):
+        await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(model_ids=["stays", "joins"]),
+            user_api_key_dict=_admin(),
+        )
+
+    written = {where["model_id"]: json.loads(data["model_info"]) for where, data in prisma.model_table.updates}
+    assert written == {"joins": {"access_groups": ["prod-models"], "description": "saved while the update waited"}}
+
+
+@pytest.mark.asyncio
+async def test_an_access_group_emptied_while_the_update_retried_a_locked_row_answers_404_and_writes_nothing():
+    from fastapi import HTTPException
+
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient(
+        [],
+        deployments=[
+            _deployment(model_id="leaves", model_name="gpt-leaves"),
+            _deployment(model_id="joins", model_name="gpt-joins", access_groups=()),
+        ],
+    )
+    prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"})
+    prisma.model_table.edits_the_other_session_commits = {"leaves": {"access_groups": []}}
+    prisma.model_table.other_session_commits_after_refusals = 1
+
+    with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised:
+        await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(model_ids=["joins"]),
+            user_api_key_dict=_admin(),
+        )
+
+    assert raised.value.status_code == 404
+    assert prisma.model_table.updates == []
+
+
+def _another_session_holds_a_member_row(model_table):
+    model_table.rows_locked_by_another_session = frozenset({"leaves"})
+
+
+def _another_edit_holds_the_group(model_table):
+    model_table.group_locked_by_another_session = True
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"]
+)
+async def test_an_access_group_update_answers_409_and_writes_nothing_while_another_session_keeps_a_lock_it_needs(
+    hold,
+):
+    from fastapi import HTTPException
+
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        _rewrite_membership_once_unlocked,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient(
+        [],
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="leaves", model_name="gpt-leaves"),
+        ],
+    )
+    hold(prisma.model_table)
+    clock = _FakeClock()
+
+    with pytest.raises(HTTPException) as raised:
+        await _rewrite_membership_once_unlocked(
+            prisma_client=prisma,
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(model_ids=["stays"]),
+            retry_budget_seconds=0.5,
+            sleep=clock.sleep,
+            clock=clock,
+        )
+
+    assert raised.value.status_code == 409
+    assert 0 < clock.now <= 0.5
+    assert prisma.model_table.lock_refusals == len(clock.slept) + 1
+    assert prisma.model_table.updates == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"]
+)
+async def test_an_access_group_update_waits_out_a_lock_held_for_over_ten_seconds_and_writes_the_edit(hold):
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        _rewrite_membership_once_unlocked,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient(
+        [],
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="leaves", model_name="gpt-leaves"),
+        ],
+    )
+    hold(prisma.model_table)
+    prisma.model_table.other_session_commits_after_refusals = 20
+    clock = _FakeClock()
+
+    _, rewrites = await _rewrite_membership_once_unlocked(
+        prisma_client=prisma,
+        access_group="prod-models",
+        data=UpdateModelGroupRequest(model_ids=["stays"]),
+        sleep=clock.sleep,
+        clock=clock,
+    )
+
+    assert clock.now > 10
+    assert [model_id for model_id, _ in rewrites] == ["leaves"]
+    assert [where["model_id"] for where, _ in prisma.model_table.updates] == ["leaves"]
+
+
+@pytest.mark.asyncio
+async def test_updating_a_missing_access_group_answers_404_before_checking_the_model_names():
+    from fastapi import HTTPException
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient(
+        [], deployments=[_deployment(model_id="other", model_name="gpt-other", access_groups=("other-group",))]
+    )
+
+    with _proxy_with_stubbed_reload(prisma):
+        _serve_model_groups(proxy_server.llm_router, ["gpt-other"])
+        with pytest.raises(HTTPException) as raised:
+            await update_access_group(
+                access_group="prod-models",
+                data=UpdateModelGroupRequest(model_names=["no-such-model"]),
+                user_api_key_dict=_admin(),
+            )
+
+    assert raised.value.status_code == 404
+    assert prisma.model_table.updates == []
+
+
+@pytest.mark.asyncio
+async def test_an_access_group_update_answers_500_when_the_deployments_cannot_be_read():
+    from fastapi import HTTPException
+
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient([], deployments=[_deployment(model_id="stays")])
+    prisma.model_table.find_many = AsyncMock(side_effect=RuntimeError("database unreachable"))
+
+    with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised:
+        await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(model_ids=["stays"]),
+            user_api_key_dict=_admin(),
+        )
+
+    assert raised.value.status_code == 500
+    assert prisma.model_table.updates == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("model_ids", [["no-such-deployment"], ["joins", "no-such-deployment"]])
+async def test_updating_an_access_group_with_an_unknown_model_id_writes_no_deployment(model_ids):
+    from fastapi import HTTPException
+
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient(
+        [],
+        deployments=[_deployment(model_id="stays"), _deployment(model_id="joins", access_groups=())],
+    )
+
+    with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised:
+        await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(model_ids=model_ids),
+            user_api_key_dict=_admin(),
+        )
+
+    assert raised.value.status_code == 400
+    assert prisma.model_table.updates == []
+
+
+@pytest.mark.asyncio
+async def test_updating_an_access_group_no_deployment_carries_answers_404_and_writes_nothing():
+    from fastapi import HTTPException
+
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient([], deployments=[_deployment(model_id="other", access_groups=("other-group",))])
+
+    with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised:
+        await update_access_group(
+            access_group="prod-models",
+            data=UpdateModelGroupRequest(model_ids=["other"]),
+            user_api_key_dict=_admin(),
+        )
+
+    assert raised.value.status_code == 404
+    assert prisma.model_table.updates == []
+
+
+@pytest.mark.asyncio
+async def test_updating_an_access_group_with_a_model_name_only_the_config_serves_writes_no_deployment():
+    from fastapi import HTTPException
+
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
+        update_access_group,
+    )
+    from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+        UpdateModelGroupRequest,
+    )
+
+    prisma = _FakePrismaClient(
+        [],
+        deployments=[
+            _deployment(model_id="stays", model_name="gpt-stays"),
+            _deployment(model_id="joins", model_name="gpt-joins", access_groups=()),
+        ],
+    )
+
+    with _proxy_with_stubbed_reload(prisma):
+        _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-joins", "config-only-model"])
+        with pytest.raises(HTTPException) as raised:
+            await update_access_group(
+                access_group="prod-models",
+                data=UpdateModelGroupRequest(model_names=["gpt-joins", "config-only-model"]),
+                user_api_key_dict=_admin(),
+            )
+
+    assert raised.value.status_code == 400
+    assert prisma.model_table.updates == []

tests/unit/proxy/management_endpoints/test_common_daily_activity.py

@@ -2553,6 +2553,71 @@ async def test_get_daily_activity_rejects_non_canonical_dates_before_querying(st
     mock_table.find_many.assert_not_awaited()
 
 
+def _provider_record(provider: str, *, model: str, spend: float):
+    base = vars(_spend_record("real-key", model=model, spend=spend))
+    return SimpleNamespace(**{**base, "custom_llm_provider": provider, "api_requests": 1, "successful_requests": 1})
+
+
+def test_per_row_provider_bucket_marks_subscription_backed_providers_only():
+    from litellm.proxy.management_endpoints.common_daily_activity import update_breakdown_metrics
+    from litellm.types.proxy.management_endpoints.common_daily_activity import BreakdownMetrics
+
+    breakdown = BreakdownMetrics()
+    update_breakdown_metrics(breakdown, _provider_record("chatgpt", model="gpt-5.6-terra", spend=0.0), {}, {}, {})
+    update_breakdown_metrics(breakdown, _provider_record("openai", model="gpt-5.6", spend=0.0), {}, {}, {})
+    update_breakdown_metrics(breakdown, _provider_record("chatgpt", model="gpt-5.6-sol", spend=0.0), {}, {}, {})
+
+    assert breakdown.providers["chatgpt"].metadata == {"subscription_covered": True}
+    assert "subscription_covered" not in breakdown.providers["openai"].metadata
+    assert breakdown.providers["chatgpt"].metrics.api_requests == 2
+    assert "subscription_covered" not in breakdown.models["gpt-5.6-terra"].metadata
+
+
+def test_per_row_provider_marker_keeps_existing_provider_metadata():
+    from litellm.proxy.management_endpoints.common_daily_activity import update_breakdown_metrics
+    from litellm.types.proxy.management_endpoints.common_daily_activity import BreakdownMetrics
+
+    breakdown = BreakdownMetrics()
+    update_breakdown_metrics(
+        breakdown,
+        _provider_record("chatgpt", model="gpt-5.6-terra", spend=0.0),
+        {},
+        {"chatgpt": {"region": "us"}},
+        {},
+    )
+
+    assert breakdown.providers["chatgpt"].metadata == {"region": "us", "subscription_covered": True}
+
+
+@pytest.mark.parametrize("provider_row_first", [True, False])
+def test_grouping_sets_provider_bucket_marks_subscription_backed_providers_in_any_row_order(provider_row_first):
+    from litellm.proxy.management_endpoints.common_daily_activity import (
+        _GROUP_DATE_PROVIDER,
+        _GROUP_DATE_PROVIDER_API_KEY,
+        _GROUP_GRAND_TOTAL,
+        _aggregate_grouping_sets_records_sync,
+    )
+
+    provider_rows = [
+        _grouping_row(_GROUP_DATE_PROVIDER, custom_llm_provider="chatgpt", spend=0.0),
+        _grouping_row(_GROUP_DATE_PROVIDER_API_KEY, custom_llm_provider="chatgpt", api_key="real-key", spend=0.0),
+    ]
+    ordered_rows = provider_rows if provider_row_first else list(reversed(provider_rows))
+    records = [
+        *ordered_rows,
+        _grouping_row(_GROUP_DATE_PROVIDER, custom_llm_provider="openai", spend=0.0),
+        _grouping_row(_GROUP_DATE_PROVIDER_API_KEY, custom_llm_provider="openai", api_key="real-key", spend=0.0),
+        _grouping_row(_GROUP_GRAND_TOTAL, spend=0.0),
+    ]
+
+    aggregated = _aggregate_grouping_sets_records_sync(records=records, api_key_metadata={})
+
+    providers = aggregated["results"][0].breakdown.providers
+    assert providers["chatgpt"].metadata == {"subscription_covered": True}
+    assert "real-key" in providers["chatgpt"].api_key_breakdown
+    assert "subscription_covered" not in providers["openai"].metadata
+
+
 @pytest.mark.parametrize("missing", [None, ""])
 def test_missing_dimensions_preserve_all_paginated_metrics(missing: str | None) -> None:
     records: Final = (

tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py

@@ -4195,6 +4195,65 @@ async def test_add_new_user_to_default_team_string_teams_have_no_member_budget(m
     assert mock_add.call_args.kwargs["team_id"] == "string-team"
 
 
+@pytest.mark.asyncio
+async def test_add_new_user_to_default_team_joins_teams_in_the_order_given_and_past_a_failing_team(
+    mocker: MockerFixture,
+) -> None:
+    from litellm.proxy.management_endpoints.internal_user_endpoints import (
+        add_new_user_to_default_team,
+    )
+
+    requested: Final = ("team-zeta", "team-broken", "team-mu", "team-alpha")
+    seconds_per_team: Final = {"team-zeta": 0.03, "team-broken": 0.0, "team-mu": 0.02, "team-alpha": 0.0}
+    finished: Final = mocker.MagicMock()
+
+    async def join_team(*, team_id: str, **_: object) -> None:
+        await asyncio.sleep(seconds_per_team[team_id])
+        if team_id == "team-broken":
+            raise RuntimeError("team row locked")
+        finished(team_id)
+
+    mocker.patch(
+        "litellm.proxy.management_endpoints.internal_user_endpoints._add_user_to_team",
+        side_effect=join_team,
+    )
+
+    await add_new_user_to_default_team(
+        user_id="scim-user",
+        user_email=None,
+        user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
+        teams=list(requested),
+        prisma_client=mocker.MagicMock(),
+    )
+
+    assert [call.args[0] for call in finished.call_args_list] == ["team-zeta", "team-mu", "team-alpha"]
+
+
+@pytest.mark.asyncio
+async def test_add_new_user_to_default_team_rejects_an_invalid_team_before_joining_any(
+    mocker: MockerFixture,
+) -> None:
+    from litellm.proxy.management_endpoints.internal_user_endpoints import (
+        add_new_user_to_default_team,
+    )
+
+    join_team: Final = mocker.patch(
+        "litellm.proxy.management_endpoints.internal_user_endpoints._add_user_to_team",
+        new_callable=mocker.AsyncMock,
+    )
+
+    with pytest.raises(ValueError, match="Invalid team type"):
+        await add_new_user_to_default_team(
+            user_id="new-user",
+            user_email=None,
+            user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
+            teams=["team-zeta", {"team_id": "team-mu"}],
+            prisma_client=mocker.MagicMock(),
+        )
+
+    join_team.assert_not_awaited()
+
+
 @pytest.mark.asyncio
 async def test_add_user_to_team_logs_unknown_team_at_error(mocker, caplog):
     """A default team that no longer exists makes every membership write 404.

tests/unit/proxy/management_endpoints/test_key_management_endpoints.py

@@ -13,7 +13,7 @@ import pytest
 import yaml
 from fastapi import HTTPException
 from fastapi.testclient import TestClient
-from pydantic import JsonValue
+from pydantic import JsonValue, ValidationError
 
 import litellm
 from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS
@@ -7597,6 +7597,7 @@ async def test_generate_key_with_router_settings(monkeypatch):
         "num_retries": 3,
         "model_group_retry_policy": {"gpt-4": {"RateLimitErrorRetries": 5}},
         "weights": {"gpt-4": {"weighted-id": 1}},
+        "content_policy_fallbacks": [{"gpt-4": ["gpt-4o", "gpt-4o-mini"]}],
     }
 
     request_data = GenerateKeyRequest(
@@ -7736,6 +7737,19 @@ async def test_update_key_with_router_settings(
     assert json.loads(cleared["router_settings"]) == {}
 
 
+@pytest.mark.parametrize("request_type", [GenerateKeyRequest, UpdateKeyRequest])
+@pytest.mark.parametrize("dict_target", [{"model": "gpt-4o-mini"}, {}])
+def test_a_key_content_policy_rule_can_only_name_fallback_models(
+    request_type: type[GenerateKeyRequest] | type[UpdateKeyRequest],
+    dict_target: dict[str, str],
+):
+    with pytest.raises(ValidationError, match="must be model names"):
+        request_type(
+            key="sk-key",
+            router_settings={"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", dict_target]}]},
+        )
+
+
 @pytest.mark.asyncio
 async def test_validate_max_budget():
     """

tests/unit/proxy/management_endpoints/test_model_management_endpoints.py

@@ -6508,7 +6508,11 @@ class TestStrategyRouterWriteValidation:
             assert fake.tx_obj.raw_calls == []
             return
         assert handle is fake.tx_obj.litellm_proxymodeltable
-        published.assert_awaited_once_with(redis_cache=None, object_type="litellm_proxymodeltable")
+        published.assert_awaited_once_with(
+            redis_cache=None,
+            object_type="litellm_proxymodeltable",
+            model_ids=(model_id,) if model_id else (),
+        )
         (lock_sql, lock_params), (count_sql, count_params) = fake.tx_obj.raw_calls
         assert "pg_advisory_xact_lock($1)" in lock_sql and "count" not in lock_sql
         assert lock_params == (AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY,)

tests/unit/proxy/management_endpoints/test_subscription_accounts.py

@@ -0,0 +1,438 @@
+from collections.abc import Iterator
+from datetime import datetime, timezone
+from typing import Final
+from uuid import uuid4
+
+from unittest.mock import MagicMock
+
+import pytest
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.testclient import TestClient
+from pydantic import ValidationError
+
+from litellm.models.subscription_account import LiteLLM_SubscriptionAccountTable
+from litellm.proxy._types import LiteLLM_JWTAuth, LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy.auth.auth_checks import allowed_routes_check
+from litellm.proxy.auth.route_checks import RouteChecks
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.management_endpoints.subscription_accounts import (
+    SubscriptionDeployment,
+    get_subscription_account_repository,
+    get_subscription_deployments,
+    router,
+    subscription_deployments_from_model_list,
+    subscription_deployments_of_router,
+)
+from litellm.router import Router
+from litellm.types.proxy.management_endpoints.subscription_accounts import (
+    SubscriptionAccountFeeRequest,
+    UpdateSubscriptionAccountFeeRequest,
+)
+from litellm.repositories.subscription_account_repository import (
+    DuplicateSubscriptionAccount,
+    SubscriptionAccountFeeChanges,
+    SubscriptionAccountFeeFields,
+)
+
+_ACCOUNT: Final = "org-team-workspace"
+_TEST_FEE: Final = {
+    "custom_llm_provider": "chatgpt",
+    "account_id": _ACCOUNT,
+    "monthly_fee": 25.0,
+    "currency": "USD",
+    "billing_period_start": "2026-10-01",
+    "label": "Team plan (test value)",
+}
+_TWO_DEPLOYMENTS_ONE_ACCOUNT: Final = (
+    SubscriptionDeployment(model_name="gpt-5.6-terra", custom_llm_provider="chatgpt", account_id=_ACCOUNT),
+    SubscriptionDeployment(model_name="gpt-5.6-sol", custom_llm_provider="chatgpt", account_id=_ACCOUNT),
+)
+
+
+class _FakeStore:
+    def __init__(self) -> None:
+        self.rows: dict[str, LiteLLM_SubscriptionAccountTable] = {}
+        self.written_columns: list[dict[str, float | str | None]] = []
+
+    async def list_all(self) -> list[LiteLLM_SubscriptionAccountTable]:
+        return sorted(self.rows.values(), key=lambda row: row.created_at)
+
+    async def find_by_id(self, id_value: str) -> LiteLLM_SubscriptionAccountTable | None:
+        return self.rows.get(id_value)
+
+    async def create_account(
+        self, custom_llm_provider: str, account_id: str, fee: SubscriptionAccountFeeFields, created_by: str
+    ) -> LiteLLM_SubscriptionAccountTable | DuplicateSubscriptionAccount:
+        for row in self.rows.values():
+            if (row.custom_llm_provider, row.account_id) == (custom_llm_provider, account_id):
+                return DuplicateSubscriptionAccount(custom_llm_provider=custom_llm_provider, account_id=account_id)
+        now = datetime.now(timezone.utc)
+        row = LiteLLM_SubscriptionAccountTable(
+            subscription_account_id=str(uuid4()),
+            custom_llm_provider=custom_llm_provider,
+            account_id=account_id,
+            label=fee.label,
+            monthly_fee=fee.monthly_fee,
+            currency=fee.currency,
+            billing_period_start=fee.billing_period_start,
+            created_at=now,
+            created_by=created_by,
+            updated_at=now,
+            updated_by=created_by,
+        )
+        self.rows[row.subscription_account_id] = row
+        return row
+
+    async def update_fee(
+        self, subscription_account_id: str, changes: SubscriptionAccountFeeChanges, updated_by: str
+    ) -> LiteLLM_SubscriptionAccountTable | None:
+        columns = dict(changes.columns())
+        self.written_columns.append(columns)
+        existing = self.rows.get(subscription_account_id)
+        if existing is None:
+            return None
+        updated = existing.model_copy(
+            update={**columns, "updated_by": updated_by, "updated_at": datetime.now(timezone.utc)}
+        )
+        self.rows[subscription_account_id] = updated
+        return updated
+
+    async def delete_account(self, subscription_account_id: str) -> LiteLLM_SubscriptionAccountTable | None:
+        return self.rows.pop(subscription_account_id, None)
+
+
+def _resolve_auth(request: Request) -> UserAPIKeyAuth:
+    role: Final = LitellmUserRoles(request.headers.get("x-user-role", LitellmUserRoles.PROXY_ADMIN.value))
+    return UserAPIKeyAuth(user_id="admin", user_role=role, api_key="sk-test")
+
+
+@pytest.fixture
+def deployments() -> tuple[SubscriptionDeployment, ...]:
+    return _TWO_DEPLOYMENTS_ONE_ACCOUNT
+
+
+@pytest.fixture
+def client(deployments: tuple[SubscriptionDeployment, ...]) -> Iterator[tuple[TestClient, _FakeStore]]:
+    store: Final = _FakeStore()
+    app: Final = FastAPI()
+    app.include_router(router)
+    app.dependency_overrides[get_subscription_account_repository] = lambda: store
+    app.dependency_overrides[get_subscription_deployments] = lambda: deployments
+    app.dependency_overrides[user_api_key_auth] = _resolve_auth
+    with TestClient(app) as test_client:
+        yield test_client, store
+
+
+def _usage(test_client: TestClient, start_date: str, end_date: str, **headers: str) -> dict:
+    response = test_client.get(
+        "/subscription_accounts/usage", params={"start_date": start_date, "end_date": end_date}, headers=headers
+    )
+    assert response.status_code == 200, response.text
+    return response.json()
+
+
+def test_untracked_account_lists_its_deployments_with_no_fee(client):
+    test_client, _ = client
+
+    body = _usage(test_client, "2026-10-01", "2026-10-07")
+
+    assert body["subscription_providers"] == ["chatgpt"]
+    assert body["accounts"] == [
+        {
+            "custom_llm_provider": "chatgpt",
+            "account_id": _ACCOUNT,
+            "deployments": ["gpt-5.6-sol", "gpt-5.6-terra"],
+            "fee": None,
+            "billing_periods": [],
+            "fixed_cost": None,
+        }
+    ]
+
+
+def test_fee_counts_once_per_billing_period_however_many_deployments_share_the_account(client):
+    test_client, _ = client
+    created = test_client.post("/subscription_accounts/new", json=_TEST_FEE)
+    assert created.status_code == 200, created.text
+
+    one_period = _usage(test_client, "2026-10-01", "2026-10-07")
+    two_periods = _usage(test_client, "2026-09-15", "2026-11-15")
+    between_starts = _usage(test_client, "2026-10-02", "2026-10-31")
+
+    (account,) = one_period["accounts"]
+    assert account["deployments"] == ["gpt-5.6-sol", "gpt-5.6-terra"]
+    assert account["fee"]["monthly_fee"] == 25.0
+    assert account["fee"]["currency"] == "USD"
+    assert account["fee"]["label"] == "Team plan (test value)"
+    assert account["billing_periods"] == [{"start": "2026-10-01", "end": "2026-10-31"}]
+    assert account["fixed_cost"] == 25.0
+    assert two_periods["accounts"][0]["fixed_cost"] == 50.0
+    assert [period["start"] for period in two_periods["accounts"][0]["billing_periods"]] == [
+        "2026-10-01",
+        "2026-11-01",
+    ]
+    assert between_starts["accounts"][0]["fixed_cost"] == 0.0
+    assert between_starts["accounts"][0]["billing_periods"] == []
+
+
+def test_fee_of_an_account_no_deployment_uses_stays_listed_and_counted(client):
+    test_client, _ = client
+    other = {**_TEST_FEE, "account_id": "org-other", "monthly_fee": 200.0, "currency": "EUR"}
+    assert test_client.post("/subscription_accounts/new", json=other).status_code == 200
+    assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200
+
+    body = _usage(test_client, "2026-10-01", "2026-10-07")
+
+    unused, used = body["accounts"]
+    assert (unused["account_id"], unused["deployments"], unused["fee"]["currency"]) == ("org-other", [], "EUR")
+    assert unused["fixed_cost"] == 200.0
+    assert (used["account_id"], used["fixed_cost"]) == (_ACCOUNT, 25.0)
+
+
+@pytest.mark.parametrize(
+    "deployments",
+    [(SubscriptionDeployment(model_name="gpt-5.6-terra", custom_llm_provider="chatgpt", account_id=None),)],
+)
+def test_account_not_signed_in_shows_no_account_and_takes_no_fee(client):
+    test_client, _ = client
+    assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200
+
+    body = _usage(test_client, "2026-10-01", "2026-10-07")
+
+    unsigned = next(account for account in body["accounts"] if account["account_id"] is None)
+    assert unsigned["deployments"] == ["gpt-5.6-terra"]
+    assert unsigned["fee"] is None
+    assert unsigned["fixed_cost"] is None
+
+
+def test_update_changes_the_fee_and_delete_returns_the_account_to_untracked(client):
+    test_client, store = client
+    created = test_client.post("/subscription_accounts/new", json=_TEST_FEE).json()
+    account_id = created["subscription_account_id"]
+
+    updated = test_client.post(
+        "/subscription_accounts/update",
+        json={"subscription_account_id": account_id, "monthly_fee": 30.0, "billing_period_start": "2026-10-05"},
+    )
+    assert updated.status_code == 200, updated.text
+    assert updated.json()["monthly_fee"] == 30.0
+    assert updated.json()["currency"] == "USD"
+    assert updated.json()["billing_period_start"] == "2026-10-05"
+    assert updated.json()["label"] == _TEST_FEE["label"]
+    assert _usage(test_client, "2026-10-01", "2026-10-07")["accounts"][0]["fixed_cost"] == 30.0
+
+    cleared = test_client.post(
+        "/subscription_accounts/update", json={"subscription_account_id": account_id, "label": None}
+    )
+    assert cleared.status_code == 200, cleared.text
+    assert cleared.json()["label"] is None
+    assert cleared.json()["monthly_fee"] == 30.0
+    assert store.written_columns == [{"monthly_fee": 30.0, "billing_period_start": "2026-10-05"}, {"label": None}]
+
+    unknown = test_client.post(
+        "/subscription_accounts/update", json={"subscription_account_id": "no-such-account", "monthly_fee": 1.0}
+    )
+    assert unknown.status_code == 404
+
+    deleted = test_client.post("/subscription_accounts/delete", json={"subscription_account_id": account_id})
+    assert deleted.status_code == 200, deleted.text
+    assert store.rows == {}
+    assert _usage(test_client, "2026-10-01", "2026-10-07")["accounts"][0]["fixed_cost"] is None
+
+    missing = test_client.post("/subscription_accounts/delete", json={"subscription_account_id": account_id})
+    assert missing.status_code == 404
+
+
+def test_rejects_a_second_fee_for_the_same_account(client):
+    test_client, _ = client
+    assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200
+
+    duplicate = test_client.post("/subscription_accounts/new", json={**_TEST_FEE, "monthly_fee": 99.0})
+
+    assert duplicate.status_code == 400
+    assert "/subscription_accounts/update" in duplicate.json()["detail"]["error"]
+
+
+def test_rejects_a_fee_on_a_metered_provider(client):
+    test_client, store = client
+
+    response = test_client.post("/subscription_accounts/new", json={**_TEST_FEE, "custom_llm_provider": "openai"})
+
+    assert response.status_code == 400
+    assert store.rows == {}
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        {**_TEST_FEE, "monthly_fee": 0},
+        {**_TEST_FEE, "account_id": ""},
+        {**_TEST_FEE, "currency": "usd"},
+        {**_TEST_FEE, "currency": "$"},
+        {**_TEST_FEE, "billing_period_start": "10/01/2026"},
+    ],
+)
+def test_rejects_malformed_fee_fields(client, payload):
+    test_client, store = client
+
+    assert test_client.post("/subscription_accounts/new", json=payload).status_code == 422
+    assert store.rows == {}
+
+
+def test_rejects_an_infinite_monthly_fee():
+    with pytest.raises(ValidationError):
+        SubscriptionAccountFeeRequest.model_validate({**_TEST_FEE, "monthly_fee": float("inf")})
+    with pytest.raises(ValidationError):
+        UpdateSubscriptionAccountFeeRequest.model_validate(
+            {"subscription_account_id": "fee-1", "monthly_fee": float("inf")}
+        )
+
+
+@pytest.mark.parametrize("deployments", [()])
+def test_fee_stays_listed_after_the_last_subscription_deployment_is_removed(client):
+    test_client, _ = client
+    assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200
+
+    body = _usage(test_client, "2026-10-01", "2026-10-07")
+
+    (account,) = body["accounts"]
+    assert (account["account_id"], account["deployments"], account["fixed_cost"]) == (_ACCOUNT, [], 25.0)
+
+
+def test_usage_rejects_a_malformed_or_inverted_range(client):
+    test_client, _ = client
+
+    malformed = [
+        test_client.get("/subscription_accounts/usage", params={"start_date": start, "end_date": "2026-10-07"})
+        for start in ("2026-10-1", "20261001", "2026-W40-1", "2026-13-01")
+    ]
+    inverted = test_client.get(
+        "/subscription_accounts/usage", params={"start_date": "2026-10-07", "end_date": "2026-10-01"}
+    )
+
+    assert [response.status_code for response in malformed] == [400, 400, 400, 400]
+    assert inverted.status_code == 400
+
+
+def test_viewer_reads_usage_but_cannot_write(client):
+    test_client, store = client
+    viewer = {"x-user-role": LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value}
+
+    assert _usage(test_client, "2026-10-01", "2026-10-07", **viewer)["accounts"][0]["account_id"] == _ACCOUNT
+    assert test_client.post("/subscription_accounts/new", json=_TEST_FEE, headers=viewer).status_code == 403
+    assert store.rows == {}
+
+
+_SUBSCRIPTION_ROUTES: Final = (
+    "/subscription_accounts/usage",
+    "/subscription_accounts/new",
+    "/subscription_accounts/update",
+    "/subscription_accounts/delete",
+)
+
+
+def test_routes_follow_the_admin_route_policy():
+    jwt_admin_defaults = LiteLLM_JWTAuth()
+
+    for route in _SUBSCRIPTION_ROUTES:
+        assert allowed_routes_check(
+            user_role=LitellmUserRoles.PROXY_ADMIN, user_route=route, litellm_proxy_roles=jwt_admin_defaults
+        )
+        assert RouteChecks.is_management_route(route)
+
+
+def test_viewer_route_gate_reads_usage_and_blocks_writes():
+    viewer_role: Final = LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value
+    get_usage = MagicMock(spec=Request, method="GET", query_params={})
+    post_new = MagicMock(spec=Request, method="POST", query_params={})
+
+    RouteChecks._check_proxy_admin_viewer_access(
+        route="/subscription_accounts/usage", _user_role=viewer_role, request_data={}, request=get_usage
+    )
+    with pytest.raises(HTTPException) as denied:
+        RouteChecks._check_proxy_admin_viewer_access(
+            route="/subscription_accounts/new", _user_role=viewer_role, request_data=_TEST_FEE, request=post_new
+        )
+    assert denied.value.status_code == 403
+
+
+def test_internal_user_cannot_read_subscription_accounts(client):
+    test_client, _ = client
+
+    response = test_client.get(
+        "/subscription_accounts/usage",
+        params={"start_date": "2026-10-01", "end_date": "2026-10-07"},
+        headers={"x-user-role": LitellmUserRoles.INTERNAL_USER.value},
+    )
+
+    assert response.status_code == 403
+
+
+def test_deployments_take_the_account_of_their_own_login():
+    model_list = [
+        {"model_name": "gpt-5.6-terra", "litellm_params": {"model": "chatgpt/gpt-5.6-terra"}},
+        {"model_name": "gpt-5.6-sol", "litellm_params": {"model": "chatgpt/gpt-5.6-sol"}},
+        {
+            "model_name": "gpt-5.6-luna",
+            "litellm_params": {"model": "chatgpt/gpt-5.6-luna", "chatgpt_auth_file": "/logins/other.json"},
+        },
+        {"model_name": "sonnet", "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-x"}},
+        {"model_name": "broken", "litellm_params": {"model": "no-such-provider-model"}},
+    ]
+    lookups: list[tuple[str, object]] = []
+
+    def resolver(provider: str, litellm_params) -> str | None:
+        lookups.append((provider, litellm_params.get("model")))
+        return "org-other" if litellm_params.get("chatgpt_auth_file") == "/logins/other.json" else _ACCOUNT
+
+    deployments = subscription_deployments_from_model_list(model_list, resolver)
+
+    assert deployments == (
+        *_TWO_DEPLOYMENTS_ONE_ACCOUNT,
+        SubscriptionDeployment(model_name="gpt-5.6-luna", custom_llm_provider="chatgpt", account_id="org-other"),
+    )
+    assert lookups == [
+        ("chatgpt", "chatgpt/gpt-5.6-terra"),
+        ("chatgpt", "chatgpt/gpt-5.6-sol"),
+        ("chatgpt", "chatgpt/gpt-5.6-luna"),
+    ]
+
+
+def test_deployments_list_each_router_deployment_once_by_the_name_callers_use():
+    llm_router: Final = Router(
+        model_list=[
+            {"model_name": "gpt-5.6-sol", "litellm_params": {"model": "chatgpt/gpt-5.6-sol"}},
+            {
+                "model_name": "model_name_team-1_7f3a",
+                "litellm_params": {"model": "chatgpt/gpt-5.6-terra"},
+                "model_info": {"team_id": "team-1", "team_public_model_name": "team-terra"},
+            },
+        ],
+        model_group_alias={"sol-alias": "gpt-5.6-sol"},
+    )
+
+    deployments = subscription_deployments_of_router(llm_router, lambda provider, litellm_params: _ACCOUNT)
+
+    assert sorted(deployment.model_name for deployment in deployments) == ["gpt-5.6-sol", "team-terra"]
+
+
+@pytest.mark.parametrize(
+    "deployments",
+    [
+        (
+            *_TWO_DEPLOYMENTS_ONE_ACCOUNT,
+            SubscriptionDeployment(model_name="gpt-5.6-luna", custom_llm_provider="chatgpt", account_id="org-other"),
+        )
+    ],
+)
+def test_two_logins_each_count_their_own_fee_once(client):
+    test_client, _ = client
+    other = {**_TEST_FEE, "account_id": "org-other", "monthly_fee": 40.0}
+    assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200
+    assert test_client.post("/subscription_accounts/new", json=other).status_code == 200
+
+    body = _usage(test_client, "2026-10-01", "2026-10-07")
+
+    assert [(a["account_id"], a["deployments"], a["fixed_cost"]) for a in body["accounts"]] == [
+        ("org-other", ["gpt-5.6-luna"], 40.0),
+        (_ACCOUNT, ["gpt-5.6-sol", "gpt-5.6-terra"], 25.0),
+    ]

tests/unit/proxy/management_endpoints/test_ui_sso.py

@@ -4049,8 +4049,8 @@ class TestCLIKeyRegenerationFlow:
         assert (
             '<select id="team_id" name="team_id" required>'
             '<option value="" disabled selected>Choose a team</option>'
-            '<option value="team-ops">Ops &lt;eu&gt; (team-ops)</option>'
             '<option value="team-claude">team-claude</option>'
+            '<option value="team-ops">Ops &lt;eu&gt; (team-ops)</option>'
             '<option value="team-ops-us">Ops &lt;eu&gt; (team-ops-us)</option>'
             "</select>"
         ) in page
@@ -4530,6 +4530,39 @@ class TestCLIKeyRegenerationFlow:
             == ()
         )
 
+    @pytest.mark.asyncio
+    async def test_fetch_cli_sso_team_details_follows_the_user_record_order(self):
+        """The CLI login binds to the first detail, so it must be the first team on the user record.
+
+        Postgres returns the rows in whatever order the table holds them, which every write to a
+        team row changes, so the bound team used to flip between sign-ins.
+        """
+        from litellm.proxy.management_endpoints.ui_sso import (
+            fetch_cli_sso_team_details,
+        )
+
+        def team_row(team_id: str) -> MagicMock:
+            row: Final = MagicMock()
+            row.model_dump.return_value = {
+                "team_id": team_id,
+                "team_alias": None,
+                "models": [],
+                "litellm_model_table": None,
+            }
+            return row
+
+        prisma_client: Final = MagicMock()
+        prisma_client.db.litellm_teamtable.find_many = AsyncMock(
+            return_value=[team_row("team-alpha"), team_row("team-zeta")]
+        )
+
+        details: Final = await fetch_cli_sso_team_details(
+            prisma_client=prisma_client,
+            teams=["team-zeta", "team-gone", "team-alpha", "team-zeta"],
+        )
+
+        assert [detail.team_id for detail in details] == ["team-zeta", "team-alpha"]
+
     @pytest.mark.asyncio
     async def test_cli_poll_key_mints_jwt_with_selected_team_grants(self):
         """The selected team's grants must reach the mint, not just its alias."""
@@ -6692,72 +6725,57 @@ class TestPKCEFunctionality:
 class TestAddMissingTeamMember:
     """Tests for the add_missing_team_member function"""
 
+    @staticmethod
+    def _recording_add() -> tuple[
+        list[tuple[str, str | None]],
+        Callable[[str, NewUserResponse | LiteLLM_UserTable], Awaitable[None]],
+    ]:
+        added: Final[list[tuple[str, str | None]]] = []
+
+        async def add(team_id: str, user_info: NewUserResponse | LiteLLM_UserTable) -> None:
+            added.append((team_id, user_info.user_id))
+
+        return added, add
+
     @pytest.mark.asyncio
     async def test_add_missing_team_member_with_new_user_response_teams_none(self):
-        """
-        Bug reproduction: When a NewUserResponse has teams=None (new SSO user),
-        add_missing_team_member() should still add the user to the SSO teams.
-
-        Currently FAILS: The function returns early when teams is None.
-        """
+        """A new SSO user's NewUserResponse has teams=None, and the user must still be added to the SSO teams."""
         from litellm.proxy._types import NewUserResponse
         from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member
 
-        # Simulate a new SSO user - NewUserResponse has teams=None by default
-        new_user = NewUserResponse(
-            user_id="new-sso-user-123",
-            key="sk-xxxxx",
-            teams=None,  # This is the default for NewUserResponse
+        new_user = NewUserResponse(user_id="new-sso-user-123", key="sk-xxxxx", teams=None)
+        added, add = self._recording_add()
+
+        await add_missing_team_member(
+            user_info=new_user,
+            sso_teams=["team-from-entra-1", "team-from-entra-2"],
+            add_team_member=add,
         )
 
-        sso_teams = ["team-from-entra-1", "team-from-entra-2"]
-
-        with patch(
-            "litellm.proxy.management_endpoints.ui_sso.create_team_member_add_task"
-        ) as mock_add_task:
-            mock_add_task.return_value = AsyncMock()
-
-            await add_missing_team_member(user_info=new_user, sso_teams=sso_teams)
-
-            # Bug: This assertion currently FAILS - no teams are added
-            # because function returns early when teams is None
-            assert (
-                mock_add_task.call_count == 2
-            ), f"Expected 2 calls to add user to teams, but got {mock_add_task.call_count}"
-            called_team_ids = [call.args[0] for call in mock_add_task.call_args_list]
-            assert set(called_team_ids) == {
-                "team-from-entra-1",
-                "team-from-entra-2",
-            }
+        assert added == [
+            ("team-from-entra-1", "new-sso-user-123"),
+            ("team-from-entra-2", "new-sso-user-123"),
+        ]
 
     @pytest.mark.asyncio
     async def test_add_missing_team_member_with_litellm_user_table_empty_teams(self):
-        """
-        Control test: When a LiteLLM_UserTable has teams=[] (existing user, no teams),
-        add_missing_team_member() should add the user to SSO teams.
-
-        This test PASSES because LiteLLM_UserTable defaults teams to [] not None.
-        """
+        """An existing user with teams=[] is added to every SSO team."""
         from litellm.proxy._types import LiteLLM_UserTable
         from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member
 
-        # Existing user has teams=[] by default (not None)
-        existing_user = LiteLLM_UserTable(
-            user_id="existing-user-456",
-            teams=[],  # Empty list, not None
+        existing_user = LiteLLM_UserTable(user_id="existing-user-456", teams=[])
+        added, add = self._recording_add()
+
+        await add_missing_team_member(
+            user_info=existing_user,
+            sso_teams=["team-from-entra-1", "team-from-entra-2"],
+            add_team_member=add,
         )
 
-        sso_teams = ["team-from-entra-1", "team-from-entra-2"]
-
-        with patch(
-            "litellm.proxy.management_endpoints.ui_sso.create_team_member_add_task"
-        ) as mock_add_task:
-            mock_add_task.return_value = AsyncMock()
-
-            await add_missing_team_member(user_info=existing_user, sso_teams=sso_teams)
-
-            # This PASSES - teams are added because teams=[] not None
-            assert mock_add_task.call_count == 2
+        assert added == [
+            ("team-from-entra-1", "existing-user-456"),
+            ("team-from-entra-2", "existing-user-456"),
+        ]
 
     @pytest.mark.asyncio
     async def test_add_user_to_teams_from_sso_response_new_user(self):
@@ -6800,96 +6818,160 @@ class TestAddMissingTeamMember:
 
     @pytest.mark.asyncio
     async def test_sso_first_login_full_flow_adds_user_to_teams(self):
-        """
-        End-to-end test: Simulates complete first-time SSO login with Entra groups.
-        Verifies teams are created AND user is added as a member.
-        """
+        """A first-time SSO login with Entra groups adds the new user as a member of every group's team."""
         from litellm.proxy._types import NewUserResponse
         from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member
 
-        team_member_calls = []
+        new_user = NewUserResponse(user_id="first-time-sso-user", key="sk-xxxxx", teams=None)
+        added, add = self._recording_add()
 
-        async def track_team_member_add(team_id, user_info):
-            team_member_calls.append({"team_id": team_id, "user_id": user_info.user_id})
-
-        # New SSO user with Entra groups
-        new_user = NewUserResponse(
-            user_id="first-time-sso-user",
-            key="sk-xxxxx",
-            teams=None,  # The problematic default
+        await add_missing_team_member(
+            user_info=new_user,
+            sso_teams=["entra-team-alpha", "entra-team-beta"],
+            add_team_member=add,
         )
 
-        sso_teams = ["entra-team-alpha", "entra-team-beta"]
-
-        with patch(
-            "litellm.proxy.management_endpoints.ui_sso.create_team_member_add_task",
-            side_effect=track_team_member_add,
-        ):
-            await add_missing_team_member(user_info=new_user, sso_teams=sso_teams)
-
-        # Bug: With current code, team_member_calls will be empty
-        # After fix: Should have 2 entries
-        assert (
-            len(team_member_calls) == 2
-        ), f"Expected 2 teams to be added, but got {len(team_member_calls)}"
-        assert {c["team_id"] for c in team_member_calls} == {
-            "entra-team-alpha",
-            "entra-team-beta",
-        }
-        assert all(c["user_id"] == "first-time-sso-user" for c in team_member_calls)
+        assert added == [
+            ("entra-team-alpha", "first-time-sso-user"),
+            ("entra-team-beta", "first-time-sso-user"),
+        ]
 
     @pytest.mark.asyncio
     @pytest.mark.parametrize(
-        "user_info_factory,teams_value,expected_teams_added",
+        "user_info_factory,expected_teams_added",
         [
-            # Bug case: NewUserResponse with teams=None
             pytest.param(
                 lambda uid: NewUserResponse(user_id=uid, key="sk-xxx", teams=None),
-                None,
-                ["team-1", "team-2"],  # Should still add teams
+                ["team-1", "team-2"],
                 id="new_user_teams_none",
             ),
-            # Working case: LiteLLM_UserTable with teams=[]
             pytest.param(
                 lambda uid: LiteLLM_UserTable(user_id=uid, teams=[]),
-                [],
                 ["team-1", "team-2"],
                 id="existing_user_empty_teams",
             ),
-            # Existing user with some teams already
             pytest.param(
                 lambda uid: LiteLLM_UserTable(user_id=uid, teams=["team-1"]),
-                ["team-1"],
-                ["team-2"],  # Only missing team should be added
+                ["team-2"],
                 id="existing_user_partial_teams",
             ),
+            pytest.param(
+                lambda uid: LiteLLM_UserTable(user_id=uid, teams=["team-2", "team-1"]),
+                [],
+                id="existing_user_all_teams",
+            ),
         ],
     )
     async def test_add_missing_team_member_handles_all_user_types(
-        self, user_info_factory, teams_value, expected_teams_added
+        self, user_info_factory, expected_teams_added
     ):
-        """
-        Parametrized test ensuring add_missing_team_member works for all user types.
-        """
+        """Only the SSO teams the user is not already on are added."""
         from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member
 
-        user_info = user_info_factory("test-user-id")
-        sso_teams = ["team-1", "team-2"]
+        added, add = self._recording_add()
 
-        added_teams = []
+        await add_missing_team_member(
+            user_info=user_info_factory("test-user-id"),
+            sso_teams=["team-1", "team-2"],
+            add_team_member=add,
+        )
 
-        async def mock_create_task(team_id, user):
-            added_teams.append(team_id)
+        assert added == [(team_id, "test-user-id") for team_id in expected_teams_added]
 
-        with patch(
-            "litellm.proxy.management_endpoints.ui_sso.create_team_member_add_task",
-            side_effect=mock_create_task,
-        ):
-            await add_missing_team_member(user_info=user_info, sso_teams=sso_teams)
+    @pytest.mark.asyncio
+    async def test_add_missing_team_member_lands_teams_in_claim_order(self):
+        """user.teams is appended to as each add finishes, and the first team on it is the one a CLI
+        login binds to, so the adds must finish in the claim's order even when an earlier add is slower.
 
-        assert set(added_teams) == set(
-            expected_teams_added
-        ), f"Expected teams {expected_teams_added}, but got {added_teams}"
+        The adds used to run concurrently from a set, so they finished in an arbitrary order.
+        """
+        from litellm.proxy._types import LiteLLM_UserTable
+        from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member
+
+        claim: Final = ["team-zeta", "team-alpha", "team-held", "team-zeta", "team-mu"]
+        yields_before_finishing: Final = {"team-zeta": 3, "team-alpha": 2, "team-mu": 1}
+        finished: Final[list[str]] = []
+
+        async def slower_when_earlier(team_id: str, user_info: NewUserResponse | LiteLLM_UserTable) -> None:
+            for _ in range(yields_before_finishing[team_id]):
+                await asyncio.sleep(0)
+            finished.append(team_id)
+
+        await add_missing_team_member(
+            user_info=LiteLLM_UserTable(user_id="group-sync-user", teams=["team-held"]),
+            sso_teams=claim,
+            add_team_member=slower_when_earlier,
+        )
+
+        assert finished == ["team-zeta", "team-alpha", "team-mu"]
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        "existing,expected_teams_added",
+        [
+            pytest.param(frozenset({"team-zeta", "team-alpha"}), ["team-zeta", "team-alpha"], id="only_teams"),
+            pytest.param(None, ["grp-everyone", "team-zeta", "grp-vpn", "team-alpha"], id="lookup_failed"),
+        ],
+    )
+    async def test_add_missing_team_member_looks_up_missing_groups_once_and_adds_only_teams(
+        self, existing: frozenset[str] | None, expected_teams_added: list[str]
+    ):
+        from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member
+
+        added, add = self._recording_add()
+        lookups: Final[list[tuple[str, ...]]] = []
+
+        async def existing_team_ids(team_ids: Sequence[str]) -> frozenset[str] | None:
+            lookups.append(tuple(team_ids))
+            return existing
+
+        await add_missing_team_member(
+            user_info=LiteLLM_UserTable(user_id="group-sync-user", teams=["team-held"]),
+            sso_teams=["grp-everyone", "team-zeta", "team-held", "grp-vpn", "team-alpha"],
+            add_team_member=add,
+            existing_team_ids=existing_team_ids,
+        )
+
+        assert lookups == [("grp-everyone", "team-zeta", "grp-vpn", "team-alpha")]
+        assert added == [(team_id, "group-sync-user") for team_id in expected_teams_added]
+
+    @pytest.mark.asyncio
+    async def test_add_missing_team_member_skips_the_lookup_when_every_group_is_held(self):
+        from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member
+
+        added, add = self._recording_add()
+        lookup: Final = AsyncMock(return_value=frozenset())
+
+        await add_missing_team_member(
+            user_info=LiteLLM_UserTable(user_id="group-sync-user", teams=["team-alpha", "team-zeta"]),
+            sso_teams=["team-zeta", "team-alpha"],
+            add_team_member=add,
+            existing_team_ids=lookup,
+        )
+
+        lookup.assert_not_awaited()
+        assert added == []
+
+    @pytest.mark.asyncio
+    async def test_find_existing_team_ids_reads_the_writer_and_returns_none_when_the_db_cannot_answer(self):
+        from litellm.proxy.management_endpoints.ui_sso import find_existing_team_ids
+
+        stored_on_writer: Final = ("team-zeta", "team-alpha")
+
+        async def find_many(*, where: Mapping[str, Mapping[str, Sequence[str]]]) -> list[SimpleNamespace]:
+            return [
+                SimpleNamespace(team_id=team_id) for team_id in stored_on_writer if team_id in where["team_id"]["in"]
+            ]
+
+        prisma: Final = MagicMock()
+        prisma.writer_db.litellm_teamtable.find_many = AsyncMock(side_effect=find_many)
+        prisma.db.litellm_teamtable.find_many = AsyncMock(return_value=[])
+        with patch("litellm.proxy.proxy_server.prisma_client", prisma):
+            assert await find_existing_team_ids(("grp-vpn", "team-alpha")) == frozenset({"team-alpha"})
+            prisma.writer_db.litellm_teamtable.find_many.side_effect = RuntimeError("db unavailable")
+            assert await find_existing_team_ids(("team-alpha",)) is None
+        with patch("litellm.proxy.proxy_server.prisma_client", None):
+            assert await find_existing_team_ids(("team-alpha",)) is None
 
 
 @pytest.mark.asyncio

tests/unit/proxy/spend_tracking/test_subscription_billing_periods.py

@@ -0,0 +1,73 @@
+from datetime import date
+
+from litellm.proxy.spend_tracking.subscription_billing_periods import (
+    BillingPeriodRange,
+    billing_periods_starting_within,
+)
+
+
+def test_one_period_per_month_start_inside_the_filter():
+    periods = billing_periods_starting_within(date(2026, 8, 5), date(2026, 9, 1), date(2026, 10, 31))
+
+    assert periods == (
+        BillingPeriodRange(start=date(2026, 9, 5), end=date(2026, 10, 4)),
+        BillingPeriodRange(start=date(2026, 10, 5), end=date(2026, 11, 4)),
+    )
+
+
+def test_filter_boundaries_are_inclusive():
+    anchor = date(2026, 10, 1)
+
+    assert billing_periods_starting_within(anchor, date(2026, 10, 1), date(2026, 10, 7)) == (
+        BillingPeriodRange(start=date(2026, 10, 1), end=date(2026, 10, 31)),
+    )
+    assert billing_periods_starting_within(anchor, date(2026, 9, 25), date(2026, 10, 1)) == (
+        BillingPeriodRange(start=date(2026, 10, 1), end=date(2026, 10, 31)),
+    )
+
+
+def test_filter_between_two_period_starts_attributes_nothing():
+    assert billing_periods_starting_within(date(2026, 10, 15), date(2026, 10, 1), date(2026, 10, 7)) == ()
+
+
+def test_no_period_before_the_anchor():
+    assert billing_periods_starting_within(date(2026, 10, 1), date(2026, 8, 1), date(2026, 9, 30)) == ()
+    assert billing_periods_starting_within(date(2026, 10, 1), date(2026, 8, 1), date(2026, 10, 1)) == (
+        BillingPeriodRange(start=date(2026, 10, 1), end=date(2026, 10, 31)),
+    )
+
+
+def test_anchor_day_clamps_to_short_months_without_drifting():
+    periods = billing_periods_starting_within(date(2026, 1, 31), date(2026, 2, 1), date(2026, 5, 31))
+
+    assert [period.start for period in periods] == [
+        date(2026, 2, 28),
+        date(2026, 3, 31),
+        date(2026, 4, 30),
+        date(2026, 5, 31),
+    ]
+    assert periods[0].end == date(2026, 3, 30)
+    assert periods[-1].end == date(2026, 6, 29)
+
+
+def test_period_ends_the_day_before_the_next_start_across_a_year_boundary():
+    periods = billing_periods_starting_within(date(2025, 12, 20), date(2025, 12, 1), date(2026, 1, 31))
+
+    assert periods == (
+        BillingPeriodRange(start=date(2025, 12, 20), end=date(2026, 1, 19)),
+        BillingPeriodRange(start=date(2026, 1, 20), end=date(2026, 2, 19)),
+    )
+
+
+def test_inverted_filter_is_empty():
+    assert billing_periods_starting_within(date(2026, 1, 1), date(2026, 3, 1), date(2026, 2, 1)) == ()
+
+
+def test_the_calendar_s_last_month_ends_on_its_last_day_instead_of_failing():
+    assert billing_periods_starting_within(date(9999, 11, 15), date(9999, 11, 1), date(9999, 12, 31)) == (
+        BillingPeriodRange(start=date(9999, 11, 15), end=date(9999, 12, 14)),
+        BillingPeriodRange(start=date(9999, 12, 15), end=date(9999, 12, 31)),
+    )
+    assert billing_periods_starting_within(date(9999, 12, 1), date(9999, 12, 1), date(9999, 12, 31)) == (
+        BillingPeriodRange(start=date(9999, 12, 1), end=date(9999, 12, 31)),
+    )