Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_/jovial-archimedes-1d743b

# Conflicts:
#	tests/e2e/ui/tests/internal-user/internalUserWithTeams.spec.ts

.github/workflows/test-unit.yml

@@ -103,6 +103,7 @@ jobs:
               tests/test_litellm/completion_extras
               tests/test_litellm/compression
               tests/test_litellm/containers
+              tests/test_litellm/endpoints
               tests/test_litellm/experimental_mcp_client
               tests/test_litellm/models
               tests/test_litellm/repositories

basedpyright-code-budget.json

@@ -57,7 +57,7 @@
     "limit": 5611
   },
   "reportMissingTypeArgument": {
-    "limit": 15350
+    "limit": 15348
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -99,19 +99,19 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44368
+    "limit": 44364
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38468
+    "limit": 38465
   },
   "reportUnknownParameterType": {
-    "limit": 19665
+    "limit": 19663
   },
   "reportUnknownVariableType": {
-    "limit": 30066
+    "limit": 30064
   },
   "reportUnnecessaryCast": {
     "limit": 111

gateway/routes/allowlist.py

@@ -86,6 +86,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/comprehendmedical",
     "/cohere/",
     "/gemini/",
+    "/gigachat/",
     "/google/",
     "/vertex_ai/",
     "/vertex-ai/",

litellm-proxy-extras/litellm_proxy_extras/migrations/20260831000000_shadow_eval_typed_targets/migration.sql

@@ -0,0 +1,21 @@
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1 FROM information_schema.columns
+        WHERE table_name = 'LiteLLM_ShadowEvalJob' AND column_name = 'api_key_id'
+    ) THEN
+        ALTER TABLE "LiteLLM_ShadowEvalJob" RENAME COLUMN "api_key_id" TO "target_id";
+    END IF;
+END $$;
+
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "target_type" TEXT NOT NULL DEFAULT 'key';
+
+DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key_direction";
+
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_one_active_per_target_direction"
+    ON "LiteLLM_ShadowEvalJob"("target_type", "target_id", "direction") WHERE "stopped_at" IS NULL;
+
+DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_api_key_id_idx";
+
+CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_target_type_target_id_idx"
+    ON "LiteLLM_ShadowEvalJob"("target_type", "target_id");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession {
 model LiteLLM_ShadowEvalJob {
   id                String   @id @default(cuid())
   group_id          String   // legs of one job share this; the API's job id
-  api_key_id        String   // hashed virtual key whose traffic this leg shadows
+  target_type       String   @default("key") // key | team | user
+  target_id         String   // hashed virtual key, team_id, or user_id whose traffic this leg shadows
   router_name       String   // the auto-router under evaluation, in either direction
   direction         String   @default("forward") // forward | reverse
   baseline_model    String?  // reverse only: the fixed model the router is judged against
   judge_model       String
   shadow_percentage Float
   max_turns         Int      // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
-  max_budget        Float?   // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
+  max_budget        Float?   // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
   created_at        DateTime @default(now())
   created_by        String?
   ends_at           DateTime
@@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob {
   stopped_by        String?  // operator who stopped it early; null when it ended on its own
 
   @@index([group_id])
-  @@index([api_key_id])
+  @@index([target_type, target_id])
   @@index([created_at])
 }
 

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -512,6 +512,13 @@ class ProxyExtrasDBManager:
         try:
             import psycopg
         except ImportError:
+            logger.warning(
+                "psycopg is not installed; skipping the LiteLLM_SpendLogs "
+                "partition check. If this table is partitioned (see "
+                "db_scripts/partition_spend_logs.sql), schema reconciliation "
+                "will try to rewrite its primary key and fail. Install the "
+                "litellm[extra_proxy] extra, which now includes psycopg."
+            )
             return False
 
         cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)

litellm/__init__.py

@@ -7,6 +7,9 @@ warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*
 # Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
 # This warning can accumulate during streaming and cause memory leaks
 warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
+# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
+# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
+warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
 ### INIT VARIABLES #########################
 import threading
 import os

litellm/constants.py

@@ -484,6 +484,22 @@ FIREWORKS_AI_80_B: Final = int(os.getenv("FIREWORKS_AI_80_B", 80))
 #### Logging callback constants ####
 REDACTED_BY_LITELM_STRING: Final = "REDACTED_BY_LITELM"
 MAX_LANGFUSE_INITIALIZED_CLIENTS: Final = int(os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50))
+# Backpressure + lifetime bounds for the /v1/messages streaming relay (see
+# BaseAnthropicMessagesStreamingIterator.async_sse_wrapper). The relay queue is
+# bounded so a slow client throttles the upstream pump instead of letting it
+# buffer the whole response in memory; the detached-drain cap bounds how many
+# post-disconnect drains may run concurrently so client behavior can't create
+# unbounded worker state.
+ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE: Final = int(
+    os.getenv("ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", "1024")
+)
+# Setting this to 0 disables detached draining entirely: every post-disconnect
+# pump bills whatever partial output it has already collected and aborts the
+# upstream stream immediately, instead of continuing to drain for the real
+# terminal usage.
+ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS: Final = int(
+    os.getenv("ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", "100")
+)
 LOGGING_WORKER_CONCURRENCY: Final = int(os.getenv("LOGGING_WORKER_CONCURRENCY", 100))  # Must be above 0
 LOGGING_WORKER_MAX_QUEUE_SIZE: Final = int(os.getenv("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000))
 LOGGING_WORKER_MAX_TIME_PER_COROUTINE: Final = float(os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0))
@@ -806,6 +822,7 @@ openai_compatible_endpoints: Final[list] = [
     "https://api.meta.ai/v1",
     "https://api.cognition.ai/v1",
     "https://api.scx.ai/v1",
+    "https://gigachat.devices.sberbank.ru/api/v1",
 ]
 
 

litellm/endpoints/speech/speech_to_completion_bridge/transformation.py

@@ -1,10 +1,14 @@
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, cast
 
+from typing_extensions import NotRequired, ReadOnly, TypedDict
+
 from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
 
 if TYPE_CHECKING:
     from litellm import Logging as LiteLLMLoggingObj
-    from litellm.types.llms.openai import HttpxBinaryResponseContent
+    from litellm.types.llms.openai import ChatCompletionUserMessage, HttpxBinaryResponseContent
     from litellm.types.utils import ModelResponse
 
 
@@ -16,7 +20,42 @@ def _completion_response_cost(model_response: "ModelResponse") -> float | None:
     return response_cost if isinstance(response_cost, float) else None
 
 
+GEMINI_TTS_CHAT_AUDIO_FORMAT: Final = "pcm16"
+
+
+class ChatAudioParam(TypedDict):
+    voice: ReadOnly[str]
+    format: ReadOnly[NotRequired[str]]
+
+
 class SpeechToCompletionBridgeTransformationHandler:
+    def _chat_completion_params(self, optional_params: Mapping[str, object]) -> Mapping[str, object]:
+        return MappingProxyType(
+            {
+                param: value
+                for param, value in optional_params.items()
+                if param in OPENAI_CHAT_COMPLETION_PARAMS and param != "response_format"
+            }
+        )
+
+    def _chat_audio_format(self, model: str, optional_params: Mapping[str, object]) -> str | None:
+        if self._is_gemini_tts_model(model):
+            return GEMINI_TTS_CHAT_AUDIO_FORMAT
+        response_format: Final = optional_params.get("response_format")
+        return response_format if isinstance(response_format, str) else None
+
+    def _chat_audio_param(
+        self, model: str, voice: str | Mapping[str, object] | None, optional_params: Mapping[str, object]
+    ) -> ChatAudioParam | None:
+        if not isinstance(voice, str):
+            return None
+        audio_format: Final = self._chat_audio_format(model, optional_params)
+        if audio_format is None:
+            voice_only: Final[ChatAudioParam] = {"voice": voice}
+            return voice_only
+        audio: Final[ChatAudioParam] = {"voice": voice, "format": audio_format}
+        return audio
+
     def transform_request(
         self,
         model: str,
@@ -28,36 +67,19 @@ class SpeechToCompletionBridgeTransformationHandler:
         litellm_logging_obj: "LiteLLMLoggingObj",
         custom_llm_provider: str,
     ) -> dict:
-        passed_optional_params: Final = {}
-        for op in optional_params:
-            if op in OPENAI_CHAT_COMPLETION_PARAMS:
-                passed_optional_params[op] = optional_params[op]
-
-        if voice is not None:
-            if isinstance(voice, str):
-                passed_optional_params["audio"] = {"voice": voice}
-                if "response_format" in optional_params:
-                    passed_optional_params["audio"]["format"] = optional_params["response_format"]
-
-        return_kwargs = {
+        user_message: Final[ChatCompletionUserMessage] = {"role": "user", "content": input}
+        return_kwargs: Final = {
             "model": model,
-            "messages": [
-                {
-                    "role": "user",
-                    "content": input,
-                }
-            ],
+            "messages": [user_message],
             "modalities": ["audio"],
-            **passed_optional_params,
+            **self._chat_completion_params(optional_params),
+            "audio": self._chat_audio_param(model, voice, optional_params),
             **litellm_params,
             "headers": headers,
             "litellm_logging_obj": litellm_logging_obj,
             "custom_llm_provider": custom_llm_provider,
         }
-
-        # filter out None values
-        return_kwargs = {k: v for k, v in return_kwargs.items() if v is not None}
-        return return_kwargs
+        return {k: v for k, v in return_kwargs.items() if v is not None}
 
     def _convert_pcm16_to_wav(self, pcm_data: bytes, sample_rate: int = 24000, channels: int = 1) -> bytes:
         """

litellm/google_genai/adapters/transformation.py

@@ -108,7 +108,6 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper):
 
     def __init__(self, completion_stream: object):
         self.sent_first_chunk = False
-        # State tracking for accumulating partial tool calls
         self.accumulated_tool_calls = dict[int, _ToolCallAccumulator]()
         self._returned_response = False
         super().__init__(completion_stream)

litellm/integrations/shadow_eval_logger.py

@@ -592,7 +592,12 @@ _JOBS_CACHE_KEY: Final = "shadow_eval:active_jobs"
 
 
 class ShadowEvalLogger(CustomLogger):
-    """Fires blind pairwise shadow evaluations for keys with an active shadow-eval job."""
+    """Fires blind pairwise shadow evaluations for targets with an active shadow-eval job.
+
+    A job targets a virtual key, a team, or a user; a request qualifies for a job when
+    any of its resolved identities (key hash, team id, user id) matches the job's
+    target, so team and user jobs cover JWT-authenticated traffic, which carries no
+    key hash at all."""
 
     def __init__(
         self,
@@ -617,10 +622,10 @@ class ShadowEvalLogger(CustomLogger):
         # generation; the refill absorbs written rows and resets.
         self._job_starts: dict[str, int] = {}  # mutable-ok: per-generation counter
 
-    async def _active_jobs(self) -> Mapping[str, tuple[ActiveShadowEvalJob, ...]]:
-        """Active jobs by api_key_id, cache-first. A key holds at most one job per
-        direction, so the value is a collection. A DB fault returns empty without
-        caching, so sampling pauses for that request and the next one retries."""
+    async def _active_jobs(self) -> Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]:
+        """Active jobs by (target_type, target_id), cache-first. A target holds at most
+        one job per direction, so the value is a collection. A DB fault returns empty
+        without caching, so sampling pauses for that request and the next one retries."""
         cached: Final = await self._jobs_cache.async_get_cache(_JOBS_CACHE_KEY)
         if cached is not None:
             return cached  # pyright: ignore[reportReturnType]  # cache stores exactly this mapping shape
@@ -652,10 +657,10 @@ class ShadowEvalLogger(CustomLogger):
                 )
                 for row in grouped or []
             }
-            by_key: Final = tuple(
+            by_target: Final = tuple(
                 sorted(
                     (
-                        (str(record.api_key_id), job)
+                        ((str(record.target_type), str(record.target_id)), job)
                         for record in records or []
                         if (job := _as_active_job(record, *attempt_stats.get(str(record.id), (0, 0.0)))) is not None
                     ),
@@ -663,7 +668,7 @@ class ShadowEvalLogger(CustomLogger):
                 )
             )
             jobs: Final = MappingProxyType(
-                {key: tuple(job for _, job in group) for key, group in groupby(by_key, key=itemgetter(0))}
+                {target: tuple(job for _, job in group) for target, group in groupby(by_target, key=itemgetter(0))}
             )
             await self._jobs_cache.async_set_cache(_JOBS_CACHE_KEY, jobs)
             self._job_starts = {}  # rebind-ok: new generation, counts absorbed into the fill
@@ -720,8 +725,18 @@ class ShadowEvalLogger(CustomLogger):
             if should_redact_message_logging(dict(kwargs)):  # mutable-ok: predicate takes a plain dict
                 return
             metadata: Final = payload.get("metadata") or _EMPTY_METADATA
-            api_key_hash: Final = metadata.get("user_api_key_hash")
-            if not api_key_hash:
+            # Each identity the request resolved to is a candidate target; JWT-auth
+            # requests carry no key hash but do carry a team and user.
+            targets: Final = tuple(
+                (target_type, str(value))
+                for target_type, value in (
+                    ("key", metadata.get("user_api_key_hash")),
+                    ("team", metadata.get("user_api_key_team_id")),
+                    ("user", metadata.get("user_api_key_user_id")),
+                )
+                if value
+            )
+            if not targets:
                 return
             request_id: Final = payload.get("id") or ""
             if not request_id:
@@ -731,8 +746,11 @@ class ShadowEvalLogger(CustomLogger):
                 return  # only surfaces this table can normalize are comparable; unknown types fail closed
             if ops.wire_params and _request_mutating_guardrail_ran(request_metadata):
                 return  # the wire-body snapshot predates the rewrite; replaying it would resurrect stripped content
+            active_jobs: Final = await self._active_jobs()
             eligible: Final = self._sampled_jobs(
-                (await self._active_jobs()).get(str(api_key_hash), ()), request_metadata, request_id
+                tuple(job for target in targets for job in active_jobs.get(target, ())),
+                request_metadata,
+                request_id,
             )
             if not eligible:
                 return
@@ -1056,7 +1074,7 @@ class ShadowEvalLogger(CustomLogger):
         )
 
 
-_EMPTY_JOBS: Final[Mapping[str, tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({})
+_EMPTY_JOBS: Final[Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({})
 
 
 def _default_prisma_provider() -> "PrismaClient | None":

litellm/litellm_core_utils/get_litellm_params.py

@@ -50,6 +50,9 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "vertex_ai_project",
             "vertex_ai_location",
             "vertex_ai_credentials",
+            "gigachat_scope",
+            "gigachat_auth_url",
+            "gigachat_access_token",
             "tpm",
             "rpm",
             "itpm",

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -369,6 +369,9 @@ def get_llm_provider(
                     elif endpoint == "https://api.meta.ai/v1":
                         custom_llm_provider = "meta"
                         dynamic_api_key = get_secret_str("META_API_KEY")
+                    elif endpoint == "https://gigachat.devices.sberbank.ru/api/v1":
+                        custom_llm_provider = "gigachat"
+                        dynamic_api_key = get_secret_str("GIGACHAT_API_KEY")
                     elif (json_provider := JSONProviderRegistry.get_by_base_url(endpoint)) is not None:
                         custom_llm_provider = json_provider.slug
                         dynamic_api_key = api_key if api_key is not None else get_secret_str(json_provider.api_key_env)
@@ -867,6 +870,9 @@ def _get_openai_compatible_provider_info(
         # Manus is OpenAI compatible for responses API
         api_base = api_base or get_secret_str("MANUS_API_BASE") or "https://api.manus.im"
         dynamic_api_key = api_key or get_secret_str("MANUS_API_KEY")
+    elif custom_llm_provider == "gigachat":
+        api_base = api_base or get_secret_str("GIGACHAT_API_BASE") or "https://gigachat.devices.sberbank.ru/api/v1"
+        dynamic_api_key = api_key or get_secret_str("GIGACHAT_API_KEY")
 
     if api_base is not None and not isinstance(api_base, str):
         raise Exception(f"api base needs to be a string. api_base={api_base}")

litellm/litellm_core_utils/litellm_logging.py

@@ -2141,6 +2141,9 @@ class Logging(LiteLLMLoggingBaseClass):
 
             logging_result: Final = self.normalize_logging_result(result=result)
 
+            if isinstance(result, Response) and isinstance(logging_result, (ModelResponse, EmbeddingResponse)):
+                result = logging_result
+
             if standard_logging_object is None and result is not None and self.stream is not True:
                 if self._is_recognized_call_type_for_logging(logging_result=logging_result) or isinstance(
                     logging_result, (dict, list)
@@ -6152,7 +6155,10 @@ def get_standard_logging_object_payload(
 
 def emit_standard_logging_payload(payload: StandardLoggingPayload):
     if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
-        print(json.dumps(payload, indent=4), flush=True)  # noqa: T201
+        try:
+            print(json.dumps(payload, indent=4, default=str), flush=True)  # noqa: T201
+        except Exception as e:  # noqa: BLE001 # Safe catch-all for verbose logging
+            verbose_logger.exception("Error serializing standard logging payload for debug output: %s", e)
 
 
 def get_standard_logging_metadata(

litellm/litellm_core_utils/model_response_utils.py

@@ -144,7 +144,6 @@ def _is_choice_non_empty(choice: StreamingChoices) -> bool:
     # Check model_extra for dynamically added fields on the choice
     choice_extra_fields: Final[Mapping[str, object]] = choice.model_extra or {}
     for extra_field_name, extra_field_value in choice_extra_fields.items():
-        # Skip certain structural fields that are just default/None placeholders
         if extra_field_name == "index" and extra_field_value == 0:
             continue
         if extra_field_name in {"finish_reason", "logprobs"} and extra_field_value is None:
@@ -192,7 +191,6 @@ def _is_delta_non_empty(delta: Delta) -> bool:
     # Check model_extra for dynamically added fields (this is where Pydantic stores them)
     delta_extra_fields: Final[Mapping[str, object]] = delta.model_extra or {}
     for extra_field_value in delta_extra_fields.values():
-        # Even structural fields are meaningful if they have actual content
         if _has_meaningful_content(extra_field_value):
             return True
 

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -859,12 +859,28 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     @staticmethod
     def _image_sources(block: Mapping[str, object]) -> tuple[str, ...]:
+        """Normalize an Anthropic image block into strings a guardrail can read.
+
+        base64 becomes a data URI so the format travels with the payload, which is what
+        the OpenAI path already puts in this field. A file source yields nothing: those
+        bytes live behind the Files API and this extractor has no client to fetch them.
+        """
         source: Final = block.get("source")
         if not isinstance(source, Mapping):
             return ()
-        # Could be base64 or url
+
+        source_type: Final = source.get("type")
+        if source_type == "url":
+            url: Final = source.get("url")
+            return (url,) if isinstance(url, str) and url else ()
+
         data: Final = source.get("data")
-        return (data,) if data else ()
+        if not isinstance(data, str) or not data:
+            return ()
+        media_type: Final = source.get("media_type")
+        if isinstance(media_type, str) and media_type:
+            return (f"data:{media_type};base64,{data}",)
+        return (data,)
 
     async def _apply_guardrail_responses_to_input(
         self,

litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py

@@ -8,6 +8,10 @@ import httpx
 from pydantic import TypeAdapter
 from typing_extensions import TypedDict
 
+from litellm.constants import (
+    ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS,
+    ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE,
+)
 from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
@@ -21,6 +25,9 @@ from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
 
 GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging()
 
+_UPSTREAM_PUMP_TASKS: Final[set[asyncio.Task[None]]] = set()  # mutable-ok: stdlib strong-ref set for pump tasks
+_DETACHED_STREAM_DRAINS: Final[set[asyncio.Task[None]]] = set()  # mutable-ok: bounded strong-ref set, detached drains
+
 INCOMPLETE_STREAM_ERROR_MESSAGE: Final = (
     "Provider stream ended before emitting a message_stop event; "
     "the response is incomplete and any partial content (e.g. tool_use input JSON) may be truncated."
@@ -133,6 +140,34 @@ def _is_terminal_stream_chunk(chunk: object) -> bool:
     return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk)
 
 
+def _try_claim_detached_drain_slot() -> bool:
+    """Claim a detached-drain slot for the current task, bounding concurrency.
+
+    Returns True if a slot was claimed (the caller may keep draining upstream
+    for billing) or False if the cap is already reached (the caller should stop
+    and bill what it has). Only touched from the event loop, so the check +
+    insert need no lock.
+    """
+    if len(_DETACHED_STREAM_DRAINS) >= ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS:
+        return False
+    current_task: Final = asyncio.current_task()
+    if current_task is not None:
+        _DETACHED_STREAM_DRAINS.add(current_task)
+        current_task.add_done_callback(_DETACHED_STREAM_DRAINS.discard)
+    return True
+
+
+def _exception_left_unconsumed(queue: "asyncio.Queue[bytes | None | BaseException]", exc: BaseException) -> bool:
+    """After client detach the relay never reads the queue again, so drain it here.
+
+    The forwarded exception still sitting in the queue means the relay tore
+    down before re-raising it, so the proxy's failure handling never ran and
+    the caller must salvage spend itself.
+    """
+    remaining: Final = tuple(queue.get_nowait() for _ in range(queue.qsize()))
+    return any(item is exc for item in remaining)
+
+
 def _sse_event(event_type: str, payload: Mapping[str, object]) -> bytes:
     return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode()
 
@@ -414,17 +449,167 @@ class BaseAnthropicMessagesStreamingIterator:
 
     async def async_sse_wrapper(
         self,
-        completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | dict],
+        completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | Mapping[str, object]],
     ) -> AsyncIterator[bytes]:
         """
         Generic async SSE wrapper that converts streaming chunks to SSE format
         and handles logging.
 
+        The upstream read runs in a detached background task (``_pump_upstream``)
+        so that a client disconnect tears down only this client-facing generator,
+        never the upstream drain + billing. The provider (e.g. Bedrock) keeps
+        generating and billing the full response regardless of the client, so
+        draining it to completion is what lets spend tracking see the real
+        terminal ``message_delta`` / ``message_stop`` usage instead of a
+        truncated placeholder count.
+
+        Chunks reach the client through a bounded queue. While the client is
+        connected the pump blocks on a full queue (racing the disconnect
+        signal), so a slow reader throttles the upstream read exactly as the old
+        direct ``yield`` did instead of letting the whole response buffer in
+        memory. Once the client goes away the pump stops enqueueing and only
+        keeps a single ``collected_chunks`` copy for billing, and the number of
+        such post-disconnect drains running at once is capped so client behavior
+        can't create unbounded worker state; over the cap the pump bills what it
+        has rather than draining further. Detached-drain lifetime is otherwise
+        bounded by the upstream stream/read timeout.
+
+        An upstream failure (Bedrock read / decode / chunk-conversion error)
+        that happens while the client is still connected is forwarded through
+        the queue and re-raised here, so the original provider exception (and
+        its status) reaches the proxy's failure handling unchanged rather than
+        being masked by a generic incomplete-stream event.
+
         This method provides the common logic for both Anthropic and Bedrock implementations.
         """
-        collected_chunks: Final = []
-        saw_terminal_event = False
+        queue: Final[asyncio.Queue[bytes | None | BaseException]] = asyncio.Queue(
+            maxsize=ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE
+        )
+        client_detached: Final = asyncio.Event()
 
+        pump_task: Final = asyncio.create_task(self._pump_upstream_to_queue(completion_stream, queue, client_detached))
+        _UPSTREAM_PUMP_TASKS.add(pump_task)
+        pump_task.add_done_callback(_UPSTREAM_PUMP_TASKS.discard)
+
+        reached_end = False  # rebind-ok: flipped once the relay consumes the end-of-stream sentinel
+        try:
+            while True:
+                item = await queue.get()
+                if item is None:
+                    reached_end = True
+                    break
+                if isinstance(item, BaseException):
+                    raise item
+                yield item
+        finally:
+            client_detached.set()
+            if not reached_end:
+                self._dispatch_pending_deferred_logging()
+
+    def _dispatch_pending_deferred_logging(self) -> None:
+        """Fire deferred billing that a torn-down response would otherwise drop.
+
+        When the pump finishes draining while the client is still connected it
+        stores the logging coroutine for ProxyLogging._fire_deferred_stream_logging,
+        which the proxy only fires on a normally completed response: a client
+        disconnect (GeneratorExit / CancelledError) re-raises past it. Without
+        this dispatch that window loses the spend row entirely.
+        """
+        deferred_cb: Final = getattr(self.litellm_logging_obj, "_on_deferred_stream_complete", None)
+        deferred_args: Final = getattr(self.litellm_logging_obj, "_deferred_stream_complete_args", None)
+        if deferred_cb is None or deferred_args is None:
+            return
+        self.litellm_logging_obj._on_deferred_stream_complete = None
+        self.litellm_logging_obj._deferred_stream_complete_args = None
+        GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=deferred_cb(*deferred_args))
+
+    async def _bill_collected_chunks(
+        self,
+        collected_chunks: list[bytes],  # mutable-ok: SSE buffer forwarded to list-typed _handle_streaming_logging
+        *,
+        stream_teardown: bool,
+    ) -> None:
+        from litellm._logging import verbose_proxy_logger
+
+        try:
+            await self._handle_streaming_logging(collected_chunks, stream_teardown=stream_teardown)
+        except Exception as exc:  # noqa: BLE001  # billing is best-effort; never crash the pump
+            verbose_proxy_logger.warning(
+                "async_sse_wrapper billing failed after %d chunks: %s(%s)",
+                len(collected_chunks),
+                type(exc).__name__,
+                exc,
+            )
+
+    @staticmethod
+    async def _abort_upstream(
+        completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | Mapping[str, object]],
+    ) -> None:
+        """Close the upstream provider stream so it stops generating and billing."""
+        from litellm._logging import verbose_proxy_logger
+
+        try:
+            await aclose_if_supported(completion_stream)
+        except Exception as exc:  # noqa: BLE001  # abort is best-effort; log and continue
+            verbose_proxy_logger.warning(
+                "async_sse_wrapper failed to abort upstream stream: %s(%s)",
+                type(exc).__name__,
+                exc,
+            )
+
+    @staticmethod
+    async def _enqueue_for_client(
+        queue: "asyncio.Queue[bytes | None | BaseException]",
+        client_detached: "asyncio.Event",
+        item: bytes | None | BaseException,
+    ) -> bool:
+        """Deliver one item to the client, applying backpressure.
+
+        Returns True if the item was queued, False if the client disconnected
+        before there was room (the item is then dropped, since a gone client
+        can't receive it). Never blocks once the client has detached.
+        """
+        if client_detached.is_set():
+            return False
+        try:
+            queue.put_nowait(item)
+        except asyncio.QueueFull:
+            pass
+        else:
+            return True
+        put_task: Final = asyncio.ensure_future(queue.put(item))
+        detached_task: Final = asyncio.ensure_future(client_detached.wait())
+        try:
+            await asyncio.wait(frozenset((put_task, detached_task)), return_when=asyncio.FIRST_COMPLETED)
+        finally:
+            if not detached_task.done():
+                detached_task.cancel()
+        if put_task.done() and not put_task.cancelled():
+            return True
+        put_task.cancel()
+        return False
+
+    async def _pump_upstream_to_queue(
+        self,
+        completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | Mapping[str, object]],
+        queue: "asyncio.Queue[bytes | None | BaseException]",
+        client_detached: "asyncio.Event",
+    ) -> None:
+        """Drain the whole upstream into ``queue`` (backpressured) and bill once.
+
+        Runs detached so a client disconnect can't interrupt the upstream read;
+        see ``async_sse_wrapper`` for the full rationale. On a completed drain
+        the success billing (or deferred park) happens before the end-of-stream
+        sentinel is enqueued: the relay can only tear down after consuming the
+        sentinel, so its teardown can never outrun the park and get mistaken
+        for a client disconnect, and a sentinel the client never consumes falls
+        back to dispatching the parked billing here.
+        """
+        from litellm._logging import verbose_proxy_logger
+
+        collected_chunks: Final[list[bytes]] = []  # mutable-ok: SSE billing buffer appended to across the drain
+        saw_terminal_event = False  # rebind-ok: accumulates across the upstream loop
+        draining_detached = False  # rebind-ok: set once this pump claims a detached-drain slot
         try:
             async for chunk in completion_stream:
                 if self.completion_start_time is None:
@@ -432,17 +617,62 @@ class BaseAnthropicMessagesStreamingIterator:
                 saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk)
                 encoded_chunk = self._convert_chunk_to_sse_format(chunk)
                 collected_chunks.append(encoded_chunk)
-                yield encoded_chunk
-        except (GeneratorExit, asyncio.CancelledError):
-            # A client disconnect tears the generator down at the yield, so the
-            # post-loop logging below never runs and the tokens already streamed
-            # (and billed by the provider) would never reach spend tracking. See LIT-5839.
-            if collected_chunks:
-                await self._handle_streaming_logging(collected_chunks, stream_teardown=True)
-            raise
+                if not client_detached.is_set():
+                    await self._enqueue_for_client(queue, client_detached, encoded_chunk)
+                    continue
+                if not draining_detached:
+                    if not _try_claim_detached_drain_slot():
+                        verbose_proxy_logger.warning(
+                            "async_sse_wrapper: detached-drain cap (%d) reached; billing %d partial "
+                            "chunks and aborting the upstream stream to stop provider billing",
+                            ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS,
+                            len(collected_chunks),
+                        )
+                        await self._bill_collected_chunks(collected_chunks, stream_teardown=True)
+                        await self._abort_upstream(completion_stream)
+                        return
+                    draining_detached = True
+        except Exception as exc:  # noqa: BLE001  # upstream errors are handled/forwarded by _handle_pump_upstream_error
+            await self._handle_pump_upstream_error(queue, client_detached, collected_chunks, exc)
+            return
 
-        if not saw_terminal_event:
-            yield _incomplete_stream_error_sse_event()
+        if client_detached.is_set():
+            await self._bill_collected_chunks(collected_chunks, stream_teardown=True)
+            return
+        if not saw_terminal_event and not await self._enqueue_for_client(
+            queue, client_detached, _incomplete_stream_error_sse_event()
+        ):
+            await self._bill_collected_chunks(collected_chunks, stream_teardown=True)
+            return
+        await self._bill_collected_chunks(collected_chunks, stream_teardown=False)
+        if not await self._enqueue_for_client(queue, client_detached, None):
+            self._dispatch_pending_deferred_logging()
 
-        # Handle logging after all chunks are processed
-        await self._handle_streaming_logging(collected_chunks)
+    async def _handle_pump_upstream_error(
+        self,
+        queue: "asyncio.Queue[bytes | None | BaseException]",
+        client_detached: "asyncio.Event",
+        collected_chunks: list[bytes],  # mutable-ok: SSE buffer forwarded to list-typed _bill_collected_chunks
+        exc: BaseException,
+    ) -> None:
+        """Forward a provider error to a still-connected client, else salvage partial spend.
+
+        Handing the original exception to the client-facing generator lets it
+        re-raise so the proxy's failure handling keeps the provider status and
+        owns logging (no success-bill). If the client already went away, or
+        disconnects before ever consuming the queued exception, no failure hook
+        runs, so bill the partial instead of dropping the request.
+        """
+        from litellm._logging import verbose_proxy_logger
+
+        if not client_detached.is_set() and await self._enqueue_for_client(queue, client_detached, exc):
+            await client_detached.wait()
+            if not _exception_left_unconsumed(queue, exc):
+                return
+        verbose_proxy_logger.warning(
+            "async_sse_wrapper upstream pump failed after client disconnect (%d chunks): %s(%s)",
+            len(collected_chunks),
+            type(exc).__name__,
+            exc,
+        )
+        await self._bill_collected_chunks(collected_chunks, stream_teardown=True)

litellm/llms/bedrock/realtime/handler.py

@@ -7,13 +7,18 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic.
 import asyncio
 import contextlib
 import json
+from collections.abc import AsyncIterator, Mapping
 from typing import Final, Protocol
 
 from pydantic import JsonValue, TypeAdapter
 
+import litellm
 from litellm._logging import _redact_string, verbose_proxy_logger
 from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
+from litellm.types.llms.openai import OpenAIRealtimeEvents
 from litellm.types.realtime import RealtimeResponseTransformInput
 
 from ..base_aws_llm import BaseAWSLLM
@@ -32,6 +37,17 @@ def _json_str(value: JsonValue) -> str | None:
     return value if isinstance(value, str) else None
 
 
+def _should_log_event(openai_message: Mapping[str, object]) -> bool:
+    logged_types: Final = (
+        litellm.logged_real_time_event_types
+        if litellm.logged_real_time_event_types is not None
+        else DefaultLoggedRealTimeEventTypes
+    )
+    if logged_types == "*":
+        return True
+    return openai_message.get("type") in logged_types
+
+
 class RealtimeClientWebSocket(Protocol):
     """The client-facing websocket surface the realtime bridge talks to."""
 
@@ -205,16 +221,22 @@ class BedrockRealtime(BaseAWSLLM):
                 )
             )
 
-            bedrock_to_client_task: Final = asyncio.create_task(
-                self._forward_bedrock_to_client(
-                    bedrock_stream,
-                    websocket,
-                    transformation_config,
-                    model,
-                    logging_obj,
-                    session_state,
+            async def forward_bedrock_and_collect_logged_events() -> tuple[OpenAIRealtimeEvents, ...]:
+                return tuple(
+                    [
+                        event
+                        async for event in self._forward_bedrock_to_client(
+                            bedrock_stream,
+                            websocket,
+                            transformation_config,
+                            model,
+                            logging_obj,
+                            session_state,
+                        )
+                    ]
                 )
-            )
+
+            bedrock_to_client_task: Final = asyncio.create_task(forward_bedrock_and_collect_logged_events())
 
             # Wait for both tasks to complete
             await asyncio.gather(
@@ -223,6 +245,27 @@ class BedrockRealtime(BaseAWSLLM):
                 return_exceptions=True,
             )
 
+            forwarded_logged_events: Final = (
+                bedrock_to_client_task.result()
+                if not bedrock_to_client_task.cancelled() and bedrock_to_client_task.exception() is None
+                else ()
+            )
+            logged_events: Final = (
+                *forwarded_logged_events,
+                *(
+                    leftover_event
+                    for leftover_event in transformation_config.leftover_usage_done_events()
+                    if _should_log_event(leftover_event)
+                ),
+            )
+            if logged_events:
+                GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
+                    logging_obj.dispatch_success_handlers(
+                        list(logged_events),  # mutable-ok: realtime spend logging requires a list result
+                        prefer_async_handlers=True,
+                    )
+                )
+
         except Exception as e:
             verbose_proxy_logger.exception("Error in BedrockRealtime.async_realtime: %s", e)
             try:
@@ -304,8 +347,8 @@ class BedrockRealtime(BaseAWSLLM):
         model: str,
         logging_obj: LiteLLMLogging,
         session_state: RealtimeResponseTransformInput,
-    ):
-        """Forward messages from Bedrock stream to client WebSocket."""
+    ) -> AsyncIterator[OpenAIRealtimeEvents]:
+        """Forward messages from Bedrock to the client, yielding the ones to record for spend logging."""
         try:
             while True:
                 # Receive from Bedrock
@@ -353,11 +396,14 @@ class BedrockRealtime(BaseAWSLLM):
                     )
 
                     # Send transformed messages to client
-                    openai_messages = transformed_response.get("response", [])
+                    response_value = transformed_response["response"]
+                    openai_messages = response_value if isinstance(response_value, list) else (response_value,)
                     for openai_message in openai_messages:
                         message_json = json.dumps(openai_message)
                         await client_ws.send_text(message_json)
                         verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200])
+                        if _should_log_event(openai_message):
+                            yield openai_message
 
         except Exception as e:
             verbose_proxy_logger.debug("Bedrock to client forwarding ended: %s", e, exc_info=True)

litellm/llms/bedrock/realtime/transformation.py

@@ -7,7 +7,7 @@ Transforms between OpenAI Realtime API format and Bedrock Nova Sonic format.
 import base64
 import json
 import uuid as uuid_lib
-from typing import Any, Final
+from typing import Any, Final, cast
 
 from pydantic import BaseModel
 
@@ -20,29 +20,54 @@ from litellm.types.llms.openai import (
     OpenAIRealtimeContentPartDone,
     OpenAIRealtimeDoneEvent,
     OpenAIRealtimeEvents,
+    OpenAIRealtimeInputAudioBufferSpeechEvent,
+    OpenAIRealtimeInputAudioTranscriptionCompleted,
+    OpenAIRealtimeInputAudioTranscriptionDelta,
     OpenAIRealtimeOutputItemDone,
     OpenAIRealtimeResponseAudioDone,
     OpenAIRealtimeResponseContentPartAdded,
     OpenAIRealtimeResponseDelta,
     OpenAIRealtimeResponseDoneObject,
     OpenAIRealtimeResponseTextDone,
+    OpenAIRealtimeResponseUsage,
     OpenAIRealtimeStreamResponseBaseObject,
     OpenAIRealtimeStreamResponseOutputItemAdded,
     OpenAIRealtimeStreamSession,
     OpenAIRealtimeStreamSessionEvents,
+    OpenAIRealtimeUsageTokenDetails,
 )
 from litellm.types.realtime import (
     ALL_DELTA_TYPES,
     RealtimeResponseTransformInput,
     RealtimeResponseTypedDict,
 )
-from litellm.utils import get_empty_usage
 
 
 class BedrockContentEnd(BaseModel):
     stopReason: str | None = None
 
 
+class BedrockUsageTokenDetails(BaseModel):
+    speechTokens: int = 0
+    textTokens: int = 0
+
+
+class BedrockUsageDetailsTotal(BaseModel):
+    input: BedrockUsageTokenDetails = BedrockUsageTokenDetails()
+    output: BedrockUsageTokenDetails = BedrockUsageTokenDetails()
+
+
+class BedrockUsageDetails(BaseModel):
+    total: BedrockUsageDetailsTotal = BedrockUsageDetailsTotal()
+
+
+class BedrockUsageEvent(BaseModel):
+    totalInputTokens: int = 0
+    totalOutputTokens: int = 0
+    totalTokens: int = 0
+    details: BedrockUsageDetails = BedrockUsageDetails()
+
+
 TRIGGER_AUDIO_SAMPLE_RATE_HERTZ: Final = 16000
 TRIGGER_AUDIO_BYTES_PER_SECOND: Final = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ * 2
 TRIGGER_LEADING_SILENCE: Final = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND // 2)
@@ -87,6 +112,15 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         # Text configuration
         self.text_media_type = "text/plain"
 
+        # Response-stream state (Bedrock events carry no role on textOutput,
+        # so the USER/ASSISTANT split from contentStart is tracked here)
+        self._user_transcript_active = False
+        self._user_transcript_generation_stage: str | None = None
+        self._user_item_id: str | None = None
+        self._user_transcript_buffer = ""
+        self._cumulative_usage = BedrockUsageEvent()
+        self._reported_usage = BedrockUsageEvent()
+
     def validate_environment(self, headers: dict, model: str, api_key: str | None = None) -> dict:
         """Validate environment - no special validation needed for Bedrock."""
         return headers
@@ -691,6 +725,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         role: Final = content_start.get("role")
 
         if role != "ASSISTANT":
+            if role == "USER" and content_start.get("type") == "TEXT":
+                self._user_transcript_active = True
+                self._user_transcript_generation_stage = self._parse_generation_stage(
+                    content_start.get("additionalModelFields")
+                )
             return (
                 [],
                 current_response_id,
@@ -700,6 +739,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             )
 
         verbose_logger.debug("Handling ASSISTANT contentStart")
+        is_new_response: Final = current_response_id is None
 
         # Initialize IDs if needed
         if not current_response_id:
@@ -715,7 +755,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
 
         returned_messages: Final[list[OpenAIRealtimeEvents]] = []
 
-        # Send response.created
+        # Send response.created only once per response (a response can contain
+        # multiple content blocks, e.g. TEXT then AUDIO)
         response_created: Final = OpenAIRealtimeStreamResponseBaseObject(
             type="response.created",
             event_id=f"event_{uuid.uuid4()}",
@@ -727,7 +768,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
                 "conversation_id": current_conversation_id,
             },
         )
-        returned_messages.append(response_created)
+        if is_new_response:
+            returned_messages.append(response_created)
 
         # Send response.output_item.added
         output_item_added: Final = OpenAIRealtimeStreamResponseOutputItemAdded(
@@ -767,6 +809,108 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             current_delta_type,
         )
 
+    @staticmethod
+    def _parse_generation_stage(additional_model_fields: object) -> str | None:
+        if not isinstance(additional_model_fields, str):
+            return None
+        try:
+            parsed: Final = json.loads(additional_model_fields)
+        except json.JSONDecodeError:
+            return None
+        stage: Final = parsed.get("generationStage") if isinstance(parsed, dict) else None
+        return stage if isinstance(stage, str) else None
+
+    def _current_user_item_id(self, new_utterance: bool = False) -> str:
+        """Item id shared by all events of one user utterance (speech boundaries and transcript)."""
+        if new_utterance or self._user_item_id is None:
+            self._user_item_id = f"item_{uuid.uuid4()}"
+        return self._user_item_id
+
+    def transform_user_speech_event(self, is_speech_start: bool) -> tuple[OpenAIRealtimeEvents, ...]:
+        """Transform Bedrock userSpeechStart/userSpeechEnd to OpenAI speech boundary events."""
+        verbose_logger.debug("Handling userSpeech%s", "Start" if is_speech_start else "End")
+        speech_event: Final[OpenAIRealtimeInputAudioBufferSpeechEvent] = {
+            "type": "input_audio_buffer.speech_started" if is_speech_start else "input_audio_buffer.speech_stopped",
+            "event_id": f"event_{uuid.uuid4()}",
+            "item_id": self._current_user_item_id(new_utterance=is_speech_start),
+        }
+        return (speech_event,)
+
+    def transform_usage_event(self, usage_event: BedrockUsageEvent) -> None:
+        """Record Bedrock's session-cumulative usage totals for the next response.done."""
+        verbose_logger.debug("Handling usageEvent")
+        self._cumulative_usage = usage_event
+
+    def _take_usage_delta(self) -> OpenAIRealtimeResponseUsage:
+        """Usage for the response now completing: cumulative totals minus what prior response.done events reported."""
+        prior: Final = self._reported_usage
+        latest: Final = self._cumulative_usage
+        self._reported_usage = latest
+        input_details: Final[OpenAIRealtimeUsageTokenDetails] = {
+            "audio_tokens": latest.details.total.input.speechTokens - prior.details.total.input.speechTokens,
+            "text_tokens": latest.details.total.input.textTokens - prior.details.total.input.textTokens,
+            "cached_tokens": 0,
+        }
+        output_details: Final[OpenAIRealtimeUsageTokenDetails] = {
+            "audio_tokens": latest.details.total.output.speechTokens - prior.details.total.output.speechTokens,
+            "text_tokens": latest.details.total.output.textTokens - prior.details.total.output.textTokens,
+        }
+        usage_delta: Final[OpenAIRealtimeResponseUsage] = {
+            "input_tokens": latest.totalInputTokens - prior.totalInputTokens,
+            "output_tokens": latest.totalOutputTokens - prior.totalOutputTokens,
+            "total_tokens": latest.totalTokens - prior.totalTokens,
+            "input_token_details": input_details,
+            "output_token_details": output_details,
+        }
+        return usage_delta
+
+    def leftover_usage_done_events(self) -> tuple[OpenAIRealtimeEvents, ...]:
+        """Logged-only response.done for usage Bedrock reports after the final turn's contentEnd."""
+        if self._cumulative_usage == self._reported_usage:
+            return ()
+        usage: Final = self._take_usage_delta()
+        leftover_done: Final = OpenAIRealtimeDoneEvent(
+            type="response.done",
+            event_id=f"event_{uuid.uuid4()}",
+            response=OpenAIRealtimeResponseDoneObject(
+                object="realtime.response",
+                id=f"resp_{uuid.uuid4()}",
+                status="completed",
+                conversation_id=f"conv_{uuid.uuid4()}",
+                usage=dict(usage),  # mutable-ok: OpenAIRealtimeResponseDoneObject types usage as plain dict
+            ),
+        )
+        return (leftover_done,)
+
+    def transform_user_transcript_event(self, transcript: str) -> tuple[OpenAIRealtimeEvents, ...]:
+        """Transform a USER-role Bedrock textOutput (ASR transcript) to an OpenAI transcription delta."""
+        verbose_logger.debug("Handling USER textOutput (ASR transcript)")
+        delta_event: Final[OpenAIRealtimeInputAudioTranscriptionDelta] = {
+            "type": "conversation.item.input_audio_transcription.delta",
+            "event_id": f"event_{uuid.uuid4()}",
+            "item_id": self._current_user_item_id(),
+            "content_index": 0,
+            "delta": transcript,
+        }
+        if self._user_transcript_generation_stage != "SPECULATIVE":
+            self._user_transcript_buffer += transcript
+        return (delta_event,)
+
+    def user_transcript_completed_events(self) -> tuple[OpenAIRealtimeEvents, ...]:
+        """One completed event with the full transcript once the FINAL user content block ends."""
+        transcript: Final = self._user_transcript_buffer
+        if not transcript:
+            return ()
+        self._user_transcript_buffer = ""
+        completed_event: Final[OpenAIRealtimeInputAudioTranscriptionCompleted] = {
+            "type": "conversation.item.input_audio_transcription.completed",
+            "event_id": f"event_{uuid.uuid4()}",
+            "item_id": self._current_user_item_id(),
+            "content_index": 0,
+            "transcript": transcript,
+        }
+        return (completed_event,)
+
     def transform_text_output_event(
         self,
         event: dict,
@@ -985,7 +1129,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         if not current_response_id or not current_conversation_id:
             return [], None, None, None
 
-        usage_obj: Final = get_empty_usage()
+        usage: Final = self._take_usage_delta()
         response_done: Final = OpenAIRealtimeDoneEvent(
             type="response.done",
             event_id=f"event_{uuid.uuid4()}",
@@ -995,11 +1139,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
                 status="completed",
                 output=[],
                 conversation_id=current_conversation_id,
-                usage={
-                    "prompt_tokens": usage_obj.prompt_tokens,
-                    "completion_tokens": usage_obj.completion_tokens,
-                    "total_tokens": usage_obj.total_tokens,
-                },
+                usage=dict(usage),
             ),
         )
 
@@ -1042,8 +1182,6 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
 
         # Create a function call arguments done event
         # This is a custom event format that matches what clients expect
-        from typing import cast
-
         function_call_event: Final[dict[str, Any]] = {
             "type": "response.function_call_arguments.done",
             "event_id": f"event_{uuid.uuid4()}",
@@ -1194,18 +1332,26 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             returned_messages.extend(events)
 
         elif "textOutput" in event:
-            events, current_delta_chunks = self.transform_text_output_event(
-                event,
-                current_output_item_id,
-                current_response_id,
-                current_delta_chunks,
-            )
-            returned_messages.extend(events)
+            if self._user_transcript_active:
+                returned_messages.extend(self.transform_user_transcript_event(event["textOutput"].get("content", "")))
+            else:
+                events, current_delta_chunks = self.transform_text_output_event(
+                    event,
+                    current_output_item_id,
+                    current_response_id,
+                    current_delta_chunks,
+                )
+                returned_messages.extend(events)
 
         elif "audioOutput" in event:
             events = self.transform_audio_output_event(event, current_output_item_id, current_response_id)
             returned_messages.extend(events)
 
+        elif "contentEnd" in event and self._user_transcript_active:
+            self._user_transcript_active = False
+            self._user_transcript_generation_stage = None
+            returned_messages.extend(self.user_transcript_completed_events())
+
         elif "contentEnd" in event:
             events, current_delta_chunks = self.transform_content_end_event(
                 event,
@@ -1224,6 +1370,12 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
                 ) = self._response_done_events(current_response_id, current_conversation_id)
                 returned_messages.extend(done_events)
 
+        elif "userSpeechStart" in event or "userSpeechEnd" in event:
+            returned_messages.extend(self.transform_user_speech_event("userSpeechStart" in event))
+
+        elif "usageEvent" in event:
+            self.transform_usage_event(BedrockUsageEvent.model_validate(event["usageEvent"]))
+
         elif "toolUse" in event:
             events, tool_call_id, tool_name = self.transform_tool_use_event(
                 event, current_output_item_id, current_response_id

litellm/llms/gigachat/__init__.py

@@ -15,9 +15,11 @@ API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/overview
 
 from .chat.transformation import GigaChatConfig, GigaChatError
 from .embedding.transformation import GigaChatEmbeddingConfig
+from .passthrough.transformation import GigaChatPassthroughConfig
 
-__all__ = [
+__all__ = (
     "GigaChatConfig",
     "GigaChatEmbeddingConfig",
     "GigaChatError",
-]
+    "GigaChatPassthroughConfig",
+)

litellm/llms/gigachat/authenticator.py

@@ -7,6 +7,7 @@ Based on official GigaChat SDK authentication flow.
 
 import time
 import uuid
+from collections.abc import Mapping
 from typing import Final
 
 import httpx
@@ -16,7 +17,7 @@ from litellm.caching.caching import InMemoryCache
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
     HTTPHandler,
-    _get_httpx_client,
+    _get_httpx_client,  # pyright: ignore[reportPrivateUsage]  # house cached-client factory has no public alias
     get_async_httpx_client,
 )
 from litellm.secret_managers.main import get_secret_str
@@ -63,6 +64,7 @@ def get_access_token(
     credentials: str | None = None,
     scope: str | None = None,
     auth_url: str | None = None,
+    litellm_params: Mapping[str, object] | None = None,
 ) -> str:
     """
     Get valid access token, using cache if available.
@@ -78,71 +80,88 @@ def get_access_token(
     Raises:
         GigaChatAuthError: If authentication fails
     """
-    credentials = credentials or _get_credentials()
-    if not credentials:
+    if not litellm_params:
+        litellm_params = {}  # mutable-ok: empty dict default; rebind-ok: provide default
+
+    access_token: Final = litellm_params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN")
+    if access_token:
+        return access_token
+
+    effective_credentials: Final = credentials or _get_credentials()
+    if not effective_credentials:
         raise GigaChatAuthError(
             status_code=401,
             message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.",
         )
 
-    scope = scope or _get_scope()
-    auth_url = auth_url or _get_auth_url()
+    effective_scope: Final = scope or litellm_params.get("gigachat_scope") or _get_scope()
+    effective_auth_url: Final = auth_url or litellm_params.get("gigachat_auth_url") or _get_auth_url()
 
     # Check cache
-    cache_key: Final = f"gigachat_token:{credentials[:16]}"
+    cache_key: Final = f"gigachat_token:{effective_credentials[:16]}"
     cached: Final = _token_cache.get_cache(cache_key)
     if cached:
-        token, expires_at = cached
+        _token, _expires_at = cached
         # Check if token is still valid (with buffer)
-        if time.time() * 1000 < expires_at - TOKEN_EXPIRY_BUFFER_MS:
+        if time.time() * 1000 < _expires_at - TOKEN_EXPIRY_BUFFER_MS:
             verbose_logger.debug("Using cached GigaChat access token")
-            return token
+            return _token
 
     # Request new token
-    token, expires_at = _request_token_sync(credentials, scope, auth_url)
+    new_token, new_expires_at = _request_token_sync(effective_credentials, effective_scope, effective_auth_url)  # pyright: ignore[reportArgumentType]  # credential keys may be broader than str
 
-    # Cache token
-    ttl_seconds: Final = max(0, (expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000)
-    if ttl_seconds > 0:
-        _token_cache.set_cache(cache_key, (token, expires_at), ttl=ttl_seconds)
+    if new_expires_at:
+        # Cache token
+        ttl_seconds: Final = max(0, (new_expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000)
+        if ttl_seconds > 0:
+            _token_cache.set_cache(cache_key, (new_token, new_expires_at), ttl=ttl_seconds)
 
-    return token
+    return new_token
 
 
 async def get_access_token_async(
     credentials: str | None = None,
     scope: str | None = None,
     auth_url: str | None = None,
+    litellm_params: Mapping[str, object] | None = None,
 ) -> str:
     """Async version of get_access_token."""
-    credentials = credentials or _get_credentials()
-    if not credentials:
+    if not litellm_params:
+        litellm_params = {}  # mutable-ok: empty dict default; rebind-ok: provide default
+
+    access_token: Final = litellm_params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN")
+    if access_token:
+        return access_token
+
+    effective_credentials: Final = credentials or _get_credentials()
+    if not effective_credentials:
         raise GigaChatAuthError(
             status_code=401,
             message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.",
         )
 
-    scope = scope or _get_scope()
-    auth_url = auth_url or _get_auth_url()
+    effective_scope: Final = scope or litellm_params.get("gigachat_scope") or _get_scope()
+    effective_auth_url: Final = auth_url or litellm_params.get("gigachat_auth_url") or _get_auth_url()
 
     # Check cache
-    cache_key: Final = f"gigachat_token:{credentials[:16]}"
+    cache_key: Final = f"gigachat_token:{effective_credentials[:16]}"
     cached: Final = _token_cache.get_cache(cache_key)
     if cached:
-        token, expires_at = cached
-        if time.time() * 1000 < expires_at - TOKEN_EXPIRY_BUFFER_MS:
+        _token, _expires_at = cached
+        if time.time() * 1000 < _expires_at - TOKEN_EXPIRY_BUFFER_MS:
             verbose_logger.debug("Using cached GigaChat access token")
-            return token
+            return _token
 
     # Request new token
-    token, expires_at = await _request_token_async(credentials, scope, auth_url)
+    new_token, new_expires_at = await _request_token_async(effective_credentials, effective_scope, effective_auth_url)  # pyright: ignore[reportArgumentType]  # credential keys may be broader than str
 
-    # Cache token
-    ttl_seconds: Final = max(0, (expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000)
-    if ttl_seconds > 0:
-        _token_cache.set_cache(cache_key, (token, expires_at), ttl=ttl_seconds)
+    if new_expires_at:
+        # Cache token
+        ttl_seconds: Final = max(0, (new_expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000)
+        if ttl_seconds > 0:
+            _token_cache.set_cache(cache_key, (new_token, new_expires_at), ttl=ttl_seconds)
 
-    return token
+    return new_token
 
 
 def _request_token_sync(
@@ -154,7 +173,7 @@ def _request_token_sync(
     Request new access token from GigaChat OAuth endpoint (sync).
 
     Returns:
-        Tuple of (access_token, expires_at_ms)
+        tuple of (access_token, expires_at_ms)
     """
     headers: Final = {
         "Authorization": f"Basic {credentials}",
@@ -169,7 +188,7 @@ def _request_token_sync(
         client: Final = _get_http_client()
         response: Final = client.post(auth_url, headers=headers, data=data, timeout=30)
         response.raise_for_status()
-        return _parse_token_response(response)
+        return _parse_token_response(response)  # pyright: ignore[reportArgumentType]  # httpx Response may be None at type level
     except httpx.HTTPStatusError as e:
         raise GigaChatAuthError(
             status_code=e.response.status_code,
@@ -204,7 +223,7 @@ async def _request_token_async(
         )
         response: Final = await client.post(auth_url, headers=headers, data=data, timeout=30)
         response.raise_for_status()
-        return _parse_token_response(response)
+        return _parse_token_response(response)  # pyright: ignore[reportArgumentType]  # httpx Response may be None at type level
     except httpx.HTTPStatusError as e:
         raise GigaChatAuthError(
             status_code=e.response.status_code,
@@ -223,7 +242,7 @@ def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
 
     # GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at'
     access_token: Final = data.get("tok") or data.get("access_token")
-    expires_at = data.get("exp") or data.get("expires_at")
+    expires_at_raw: Final = data.get("exp") or data.get("expires_at")
 
     if not access_token:
         raise GigaChatAuthError(
@@ -232,8 +251,11 @@ def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
         )
 
     # expires_at is in milliseconds
-    if isinstance(expires_at, str):
-        expires_at = int(expires_at)
+    expires_at: int  # rebind-ok: conditionally assigned from str or int
+    if isinstance(expires_at_raw, str):
+        expires_at = int(expires_at_raw)  # rebind-ok: conditionally assigned from str or int
+    else:
+        expires_at = expires_at_raw  # pyright: ignore[reportAssignmentType]  # raw value is int or str; converted above; rebind-ok: conditionally assigned from str or int
 
     verbose_logger.debug("GigaChat access token obtained successfully")
     return access_token, expires_at

litellm/llms/gigachat/chat/__init__.py

@@ -5,8 +5,8 @@ GigaChat Chat Module
 from .streaming import GigaChatModelResponseIterator
 from .transformation import GigaChatConfig, GigaChatError
 
-__all__ = [
+__all__ = (
     "GigaChatConfig",
     "GigaChatError",
     "GigaChatModelResponseIterator",
-]
+)

litellm/llms/gigachat/chat/streaming.py

@@ -4,13 +4,15 @@ GigaChat Streaming Response Handler
 
 import json
 import uuid
+from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
+from litellm.llms.gigachat.utils import convert_usage
 from litellm.types.llms.openai import (
     ChatCompletionToolCallChunk,
     ChatCompletionToolCallFunctionChunk,
 )
-from litellm.types.utils import GenericStreamingChunk
+from litellm.types.utils import ChatCompletionUsageBlock, GenericStreamingChunk
 
 
 class GigaChatModelResponseIterator:
@@ -26,14 +28,9 @@ class GigaChatModelResponseIterator:
         self.response_iterator = self.streaming_response
         self.json_mode = json_mode
 
-    def chunk_parser(self, chunk: dict) -> GenericStreamingChunk:
+    def chunk_parser(self, chunk: Mapping[str, object]) -> GenericStreamingChunk:
         """Parse a single streaming chunk from GigaChat."""
-        text = ""
-        tool_use: ChatCompletionToolCallChunk | None = None
-        is_finished = False
-        finish_reason: str | None = None
-
-        choices: Final = chunk.get("choices", [])
+        choices: Sequence = chunk.get("choices") or ()  # mutable-ok: tuple literal as default
         if not choices:
             return GenericStreamingChunk(
                 text="",
@@ -45,40 +42,63 @@ class GigaChatModelResponseIterator:
             )
 
         choice: Final = choices[0]
-        delta: Final = choice.get("delta", {})
-        finish_reason = choice.get("finish_reason")
+        delta: Mapping[str, object] = choice.get("delta") or {}  # mutable-ok: empty dict default for get
+        chunk_finish_reason: Final = choice.get("finish_reason")
 
         # Extract text content
-        text = delta.get("content", "") or ""
+        text: Final = delta.get("content", "") or ""
+
+        usage_block: ChatCompletionUsageBlock | None = None  # rebind-ok: conditionally assigned after stop detection
+        tool_use: ChatCompletionToolCallChunk | None = None  # rebind-ok: conditionally assigned on function_call
+        finish_reason: str | None = chunk_finish_reason
 
         # Handle function_call in stream
-        if finish_reason == "function_call" and delta.get("function_call"):
-            func_call: Final = delta["function_call"]
-            args = func_call.get("arguments", {})
-
-            if isinstance(args, dict):
-                args = json.dumps(args, ensure_ascii=False)
+        raw_function_call: Final = delta.get("function_call")
+        if chunk_finish_reason == "function_call" and isinstance(raw_function_call, Mapping) and raw_function_call:
+            func_call: Final[Mapping[str, object]] = raw_function_call
+            args_raw: Final[object] = func_call.get("arguments") or {}
+            args_str: str  # rebind-ok: conditionally assigned from dict or str
+            if isinstance(args_raw, dict):
+                args_str = json.dumps(args_raw, ensure_ascii=False)  # rebind-ok: build from dict
+            else:
+                args_str = str(args_raw)
 
+            name_raw: Final = func_call.get("name")
             tool_use = ChatCompletionToolCallChunk(
                 id=f"call_{uuid.uuid4().hex[:24]}",
                 type="function",
                 function=ChatCompletionToolCallFunctionChunk(
-                    name=func_call.get("name", ""),
-                    arguments=args,
+                    name=name_raw if isinstance(name_raw, str) else "",
+                    arguments=args_str,
                 ),
                 index=0,
             )
             finish_reason = "tool_calls"
 
-        if finish_reason is not None:
-            is_finished = True
+        usage_data: Final = chunk.get("usage") or {}  # mutable-ok: empty dict default
+        if usage_data and isinstance(usage_data, dict):
+            validated_usage: Final = {k: int(v) for k, v in usage_data.items()}
+            usage = convert_usage(validated_usage)
+            _prompt_details: dict | None = (
+                usage.prompt_tokens_details.model_dump() if usage.prompt_tokens_details else None
+            )  # rebind-ok: conditional
+            _completion_details: dict | None = (
+                usage.completion_tokens_details.model_dump() if usage.completion_tokens_details else None
+            )  # rebind-ok: conditional
+            usage_block = ChatCompletionUsageBlock(  # pyright: ignore[reportCallIssue]  # TypedDict kwarg constructor
+                prompt_tokens=usage.prompt_tokens,
+                completion_tokens=usage.completion_tokens,
+                total_tokens=usage.total_tokens,
+                prompt_tokens_details=_prompt_details,
+                completion_tokens_details=_completion_details,
+            )
 
         return GenericStreamingChunk(
-            text=text,
+            text=str(text),
             tool_use=tool_use,
-            is_finished=is_finished,
+            is_finished=chunk_finish_reason is not None,
             finish_reason=finish_reason or "",
-            usage=None,
+            usage=usage_block,
             index=choice.get("index", 0),
         )
 

litellm/llms/gigachat/chat/transformation.py

@@ -4,19 +4,22 @@ GigaChat Chat Transformation
 Transforms OpenAI-format requests to GigaChat format and back.
 """
 
+from __future__ import annotations
+
 import json
 import time
 import uuid
-from collections.abc import AsyncIterator, Iterator
+from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
 
 from litellm._logging import verbose_logger
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
+from litellm.llms.gigachat.utils import convert_usage, get_api_base
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import Choices, Message, ModelResponse, Usage
+from litellm.types.utils import Choices, Message, ModelResponse
 
 from ..authenticator import get_access_token
 from ..file_handler import upload_file_sync
@@ -30,9 +33,6 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
-# GigaChat API endpoint
-GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1"
-
 
 def is_valid_json(value: str) -> bool:
     """Checks whether the value passed is a valid serialized JSON string"""
@@ -90,30 +90,30 @@ class GigaChatConfig(BaseConfig):
         api_base: str | None,
         api_key: str | None,
         model: str,
-        optional_params: dict,
-        litellm_params: dict,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
         stream: bool | None = None,
     ) -> str:
         """Get complete API URL for chat completions."""
-        base: Final = api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL
+        base: Final = get_api_base(api_base)
         return f"{base}/chat/completions"
 
     def validate_environment(
         self,
-        headers: dict,
+        headers: dict,  # mutable-ok: mutates in place per GigaChat OAuth setup
         model: str,
-        messages: list[AllMessageValues],
-        optional_params: dict,
-        litellm_params: dict,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
         api_key: str | None = None,
         api_base: str | None = None,
-    ) -> dict:
+    ) -> dict:  # mutable-ok: base class contract returns dict for httpx
         """
         Set up headers with OAuth token.
         """
         # Get access token
         credentials: Final = api_key or get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY")
-        access_token: Final = get_access_token(credentials=credentials)
+        access_token: Final = get_access_token(credentials=credentials, litellm_params=litellm_params)
 
         # Store credentials for image uploads
         self._current_credentials = credentials
@@ -125,9 +125,9 @@ class GigaChatConfig(BaseConfig):
 
         return headers
 
-    def get_supported_openai_params(self, model: str) -> list[str]:
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: base class contract returns list
         """Return list of supported OpenAI parameters."""
-        return [
+        return [  # mutable-ok: base class contract returns list
             "stream",
             "temperature",
             "top_p",
@@ -143,11 +143,11 @@ class GigaChatConfig(BaseConfig):
 
     def map_openai_params(
         self,
-        non_default_params: dict,
-        optional_params: dict,
+        non_default_params: Mapping[str, object],
+        optional_params: dict,  # mutable-ok: mutated in place per GigaChat mapping
         model: str,
         drop_params: bool,
-    ) -> dict:
+    ) -> dict:  # mutable-ok: base class contract returns dict
         """Map OpenAI parameters to GigaChat parameters."""
         for param, value in non_default_params.items():
             if param == "stream":
@@ -167,42 +167,50 @@ class GigaChatConfig(BaseConfig):
                 pass
             elif param == "tools":
                 # Convert tools to functions format
-                optional_params["functions"] = self._convert_tools_to_functions(value)
+                if isinstance(value, Sequence):
+                    optional_params["functions"] = self._convert_tools_to_functions(value)
             elif param == "tool_choice":
                 # Map OpenAI tool_choice to GigaChat function_call
-                mapped_choice = self._map_tool_choice(value)
-                if mapped_choice is not None:
-                    optional_params["function_call"] = mapped_choice
+                if isinstance(value, (str, Mapping)):
+                    mapped_choice = self._map_tool_choice(value)
+                    if mapped_choice is not None:
+                        optional_params["function_call"] = mapped_choice
             elif param == "functions":
                 optional_params["functions"] = value
             elif param == "function_call":
                 optional_params["function_call"] = value
             elif param == "response_format":
                 # Handle structured output via function calling
-                if value.get("type") == "json_schema":
+                if isinstance(value, Mapping) and value.get("type") == "json_schema":
                     json_schema = value.get("json_schema", {})
                     schema_name = json_schema.get("name", "structured_output")
                     schema = json_schema.get("schema", {})
 
-                    function_def = {
+                    function_def = {  # mutable-ok: request payload for httpx
                         "name": schema_name,
                         "description": f"Output structured response: {schema_name}",
                         "parameters": schema,
                     }
 
-                    if "functions" not in optional_params:
-                        optional_params["functions"] = []
-                    optional_params["functions"].append(function_def)
-                    optional_params["function_call"] = {"name": schema_name}
+                    existing_functions = optional_params.get("functions")
+                    optional_params["functions"] = [
+                        *(
+                            existing_functions
+                            if isinstance(existing_functions, Sequence) and not isinstance(existing_functions, str)
+                            else ()
+                        ),
+                        function_def,
+                    ]
+                    optional_params["function_call"] = {"name": schema_name}  # mutable-ok: request payload
                     optional_params["_structured_output"] = True
 
         return optional_params
 
-    def _convert_tools_to_functions(self, tools: list[dict]) -> list[dict]:
+    def _convert_tools_to_functions(self, tools: Sequence) -> Sequence[dict]:
         """Convert OpenAI tools format to GigaChat functions format."""
-        functions: Final = []
+        functions: Final[list[dict]] = []  # mutable-ok: accumulator for building functions list
         for tool in tools:
-            if tool.get("type") == "function":
+            if isinstance(tool, dict) and tool.get("type") == "function":
                 func = tool.get("function", {})
                 functions.append(
                     {
@@ -213,7 +221,7 @@ class GigaChatConfig(BaseConfig):
                 )
         return functions
 
-    def _map_tool_choice(self, tool_choice: str | dict) -> str | dict | None:
+    def _map_tool_choice(self, tool_choice: str | Mapping[str, object]) -> str | Mapping[str, object] | None:
         """
         Map OpenAI tool_choice to GigaChat function_call format.
 
@@ -246,8 +254,9 @@ class GigaChatConfig(BaseConfig):
             # OpenAI format: {"type": "function", "function": {"name": "func_name"}}
             # GigaChat format: {"name": "func_name"}
             if tool_choice.get("type") == "function":
-                func_name: Final = tool_choice.get("function", {}).get("name")
-                if func_name:
+                function_spec: Final = tool_choice.get("function")
+                func_name: Final = function_spec.get("name") if isinstance(function_spec, Mapping) else None
+                if isinstance(func_name, str) and func_name:
                     return {"name": func_name}
 
         # Default to None (don't set function_call)
@@ -273,20 +282,51 @@ class GigaChatConfig(BaseConfig):
             verbose_logger.error("Failed to upload image: %s", e)
             return None
 
+    def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[str]]:
+        """
+        Extract text and image attachments from a multimodal message content list.
+
+        Args:
+            content: List of content parts (OpenAI multimodal format)
+
+        Returns:
+            Tuple of (combined text, list of attachment file ids)
+        """
+        texts: Final[list[str]] = []  # mutable-ok: accumulator
+        attachments: Final[list[str]] = []  # mutable-ok: accumulator
+        for part in content:
+            if isinstance(part, dict):
+                if part.get("type") == "text":
+                    texts.append(part.get("text", ""))
+                elif part.get("type") == "image_url":
+                    # Extract image URL and upload to GigaChat
+                    image_url: object = part.get("image_url", {})
+                    upload_url: str
+                    if isinstance(image_url, str):
+                        upload_url = image_url
+                    else:
+                        upload_url = str(image_url.get("url", "")) if isinstance(image_url, dict) else ""
+                    if upload_url:
+                        file_id = self._upload_image(upload_url)
+                        if file_id:
+                            attachments.append(file_id)
+        text: Final = "\n".join(texts) if texts else ""
+        return text, attachments
+
     def transform_request(
         self,
         model: str,
-        messages: list[AllMessageValues],
-        optional_params: dict,
-        litellm_params: dict,
-        headers: dict,
-    ) -> dict:
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        headers: Mapping[str, object],
+    ) -> dict:  # mutable-ok: request payload sent to httpx
         """Transform OpenAI request to GigaChat format."""
         # Transform messages
         giga_messages: Final = self._transform_messages(messages)
 
         # Build request
-        request_data: Final = {
+        request_data: Final[dict[str, object]] = {
             "model": model.replace("gigachat/", ""),
             "messages": giga_messages,
         }
@@ -311,9 +351,9 @@ class GigaChatConfig(BaseConfig):
 
         return request_data
 
-    def _transform_messages(self, messages: list[AllMessageValues]) -> list[dict]:
+    def _transform_messages(self, messages: Sequence[AllMessageValues]) -> Sequence[dict]:
         """Transform OpenAI messages to GigaChat format."""
-        transformed: Final = []
+        transformed: Final[list[dict]] = []  # mutable-ok: accumulator for building transformed messages
 
         for i, msg in enumerate(messages):
             message = dict(msg)
@@ -341,24 +381,7 @@ class GigaChatConfig(BaseConfig):
             # Handle list content (multimodal) - extract text and images
             content = message.get("content")
             if isinstance(content, list):
-                texts = []
-                attachments = []
-                for part in content:
-                    if isinstance(part, dict):
-                        if part.get("type") == "text":
-                            texts.append(part.get("text", ""))
-                        elif part.get("type") == "image_url":
-                            # Extract image URL and upload to GigaChat
-                            image_url = part.get("image_url", {})
-                            if isinstance(image_url, str):
-                                url = image_url
-                            else:
-                                url = image_url.get("url", "")
-                            if url:
-                                file_id = self._upload_image(url)
-                                if file_id:
-                                    attachments.append(file_id)
-                message["content"] = "\n".join(texts) if texts else ""
+                message["content"], attachments = self._transform_list_content(content)
                 if attachments:
                     message["attachments"] = attachments
 
@@ -393,7 +416,7 @@ class GigaChatConfig(BaseConfig):
         messages: list[AllMessageValues],
         optional_params: dict,
         litellm_params: dict,
-        encoding: "tiktoken.Encoding | None",
+        encoding: tiktoken.Encoding | None,
         api_key: str | None = None,
         json_mode: bool | None = None,
     ) -> ModelResponse:
@@ -408,7 +431,7 @@ class GigaChatConfig(BaseConfig):
 
         is_structured_output: Final = optional_params.get("_structured_output", False)
 
-        choices: Final = []
+        choices: Final[list[Choices]] = []  # mutable-ok: accumulator for building response choices
         for choice in response_json.get("choices", []):
             message_data = choice.get("message", {})
             finish_reason = choice.get("finish_reason", "stop")
@@ -462,11 +485,7 @@ class GigaChatConfig(BaseConfig):
 
         # Build usage
         usage_data: Final = response_json.get("usage", {})
-        usage: Final = Usage(
-            prompt_tokens=usage_data.get("prompt_tokens", 0),
-            completion_tokens=usage_data.get("completion_tokens", 0),
-            total_tokens=usage_data.get("total_tokens", 0),
-        )
+        usage: Final = convert_usage(usage_data)
 
         model_response.id = response_json.get("id", f"chatcmpl-{uuid.uuid4().hex[:12]}")
         model_response.created = response_json.get("created", int(time.time()))

litellm/llms/gigachat/embedding/transformation.py

@@ -5,6 +5,8 @@ Transforms OpenAI /v1/embeddings format to GigaChat format.
 API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/reference/rest/post-embeddings
 """
 
+from __future__ import annotations
+
 import types
 from typing import Final
 
@@ -14,14 +16,12 @@ from litellm import LlmProviders
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
+from litellm.llms.gigachat.utils import get_api_base
 from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
 from litellm.types.utils import EmbeddingResponse
 
 from ..authenticator import get_access_token
 
-# GigaChat API endpoint
-GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1"
-
 
 class GigaChatEmbeddingError(BaseLLMException):
     """GigaChat Embedding API error."""
@@ -78,9 +78,9 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig):
         Returns provider info for GigaChat.
 
         Returns:
-            Tuple of (custom_llm_provider, api_base, dynamic_api_key)
+            tuple of (custom_llm_provider, api_base, dynamic_api_key)
         """
-        api_base = api_base or GIGACHAT_BASE_URL
+        api_base = get_api_base(api_base)
         return LlmProviders.GIGACHAT.value, api_base, api_key
 
     def get_complete_url(
@@ -93,7 +93,7 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig):
         stream: bool | None = None,
     ) -> str:
         """Get the complete URL for embeddings endpoint."""
-        base: Final = api_base or GIGACHAT_BASE_URL
+        base: Final = get_api_base(api_base)
         return f"{base}/embeddings"
 
     def transform_embedding_request(
@@ -114,14 +114,12 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig):
         """
         # Normalize input to list
         if isinstance(input, str):
-            input_list: list = [input]
-        elif isinstance(input, list):
-            input_list = input
+            input_list: list = [input]  # rebind-ok: locally scoped conversion
         else:
-            input_list = [input]
+            input_list = input
 
         # Remove gigachat/ prefix from model if present
-        model = model.removeprefix("gigachat/")
+        model = model.removeprefix("gigachat/")  # rebind-ok: parameter reassignment for normalization
 
         return {
             "model": model,
@@ -191,7 +189,7 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig):
         Set up headers with OAuth token for GigaChat.
         """
         # Get access token via OAuth
-        access_token: Final = get_access_token(api_key)
+        access_token: Final = get_access_token(credentials=api_key, litellm_params=litellm_params)
 
         default_headers: Final = {
             "Content-Type": "application/json",

litellm/llms/gigachat/file_handler.py

@@ -9,6 +9,7 @@ import base64
 import hashlib
 import re
 import uuid
+from collections.abc import Mapping
 from typing import Final
 
 from litellm._logging import verbose_logger
@@ -16,13 +17,11 @@ from litellm.llms.custom_httpx.http_handler import (
     _get_httpx_client,
     get_async_httpx_client,
 )
+from litellm.llms.gigachat.utils import get_api_base
 from litellm.types.utils import LlmProviders
 
 from .authenticator import get_access_token, get_access_token_async
 
-# GigaChat API endpoint
-GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1"
-
 # Simple in-memory cache for file IDs
 _file_cache: Final[dict[str, str]] = {}
 
@@ -82,6 +81,7 @@ def upload_file_sync(
     image_url: str,
     credentials: str | None = None,
     api_base: str | None = None,
+    litellm_params: Mapping[str, object] | None = None,
 ) -> str | None:
     """
     Upload file to GigaChat and return file_id (sync).
@@ -114,10 +114,10 @@ def upload_file_sync(
         filename: Final = f"{uuid.uuid4()}.{ext}"
 
         # Get access token
-        access_token: Final = get_access_token(credentials)
+        access_token: Final = get_access_token(credentials=credentials, litellm_params=litellm_params)
 
         # Upload to GigaChat
-        base_url: Final = api_base or GIGACHAT_BASE_URL
+        base_url: Final = get_api_base(api_base)
         upload_url: Final = f"{base_url}/files"
 
         client: Final = _get_httpx_client(params={"ssl_verify": False})
@@ -147,6 +147,7 @@ async def upload_file_async(
     image_url: str,
     credentials: str | None = None,
     api_base: str | None = None,
+    litellm_params: Mapping[str, object] | None = None,
 ) -> str | None:
     """
     Upload file to GigaChat and return file_id (async).
@@ -179,10 +180,10 @@ async def upload_file_async(
         filename: Final = f"{uuid.uuid4()}.{ext}"
 
         # Get access token
-        access_token: Final = await get_access_token_async(credentials)
+        access_token: Final = await get_access_token_async(credentials=credentials, litellm_params=litellm_params)
 
         # Upload to GigaChat
-        base_url: Final = api_base or GIGACHAT_BASE_URL
+        base_url: Final = get_api_base(api_base)
         upload_url: Final = f"{base_url}/files"
 
         client: Final = get_async_httpx_client(

litellm/llms/gigachat/passthrough/__init__.py

@@ -0,0 +1,7 @@
+"""
+GigaChat passthrough Module
+"""
+
+from .transformation import GigaChatPassthroughConfig
+
+__all__ = ("GigaChatPassthroughConfig",)

litellm/llms/gigachat/passthrough/transformation.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Final
+
+import httpx
+
+from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
+from litellm.llms.gigachat.authenticator import get_access_token
+from litellm.llms.gigachat.chat.streaming import GigaChatModelResponseIterator
+from litellm.llms.gigachat.utils import GIGACHAT_BASE_URL
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import EmbeddingResponse
+
+if TYPE_CHECKING:
+    from httpx import URL, Response
+
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.types.utils import CostResponseTypes
+
+
+class GigaChatPassthroughConfig(BasePassthroughConfig):
+    def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool:
+        return request_data.get("stream", False)
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        endpoint: str,
+        request_query_params: Mapping[str, object] | None,
+        litellm_params: Mapping[str, object],
+    ) -> tuple[URL, str]:
+        """Get complete API URL for chat completions."""
+        base_target_url: Final = self.get_api_base(api_base)
+
+        if base_target_url is None:
+            raise Exception("GigaChat api base not found")
+
+        complete_url: Final = f"{base_target_url}/{endpoint.lstrip('/')}"
+
+        return (
+            httpx.URL(complete_url),
+            base_target_url,
+        )
+
+    def validate_environment(
+        self,
+        headers: dict,  # mutable-ok: mutates in place to set OAuth headers
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict:  # mutable-ok: base class contract returns dict for httpx
+        """
+        Set up headers with OAuth token.
+        """
+        # Get access token
+        access_token: Final = get_access_token(credentials=api_key, litellm_params=litellm_params)
+
+        headers["Authorization"] = f"Bearer {access_token}"  # rebind-ok: mutating for OAuth setup
+        headers["Content-Type"] = "application/json"  # rebind-ok: mutating for OAuth setup
+        headers["Accept"] = "application/json"  # rebind-ok: mutating for OAuth setup
+
+        return headers
+
+    def logging_non_streaming_response(
+        self,
+        model: str,
+        custom_llm_provider: str,
+        httpx_response: Response,
+        request_data: Mapping[str, object],
+        logging_obj: LiteLLMLoggingObj,
+        endpoint: str,
+    ) -> CostResponseTypes | None:
+        from litellm import encoding
+        from litellm.types.utils import LlmProviders, ModelResponse
+        from litellm.utils import ProviderConfigManager
+
+        # cost tracking only for completions and embeddings
+        if "completions" in endpoint:
+            provider_chat_config: Final = ProviderConfigManager.get_provider_chat_config(
+                provider=LlmProviders(custom_llm_provider),
+                model=model,
+            )
+
+            if provider_chat_config is None:
+                raise ValueError(f"No provider config found for model: {model}")
+
+            raw_messages: Final = request_data.get("messages")
+            litellm_model_response: Final = provider_chat_config.transform_response(
+                model=model,
+                messages=list(raw_messages)
+                if isinstance(raw_messages, list)
+                else [],  # mutable-ok: transform_response wants a list
+                raw_response=httpx_response,
+                model_response=ModelResponse(),
+                logging_obj=logging_obj,
+                optional_params={},  # mutable-ok: empty dict kwarg for transform_response
+                litellm_params={},  # mutable-ok: empty dict kwarg for transform_response
+                api_key="",
+                request_data=dict(request_data),  # mutable-ok: transform_response wants a dict
+                encoding=encoding,
+            )
+
+            return litellm_model_response
+
+        if "embeddings" in endpoint:
+            provider_embedding_config: Final = ProviderConfigManager.get_provider_embedding_config(
+                provider=LlmProviders(custom_llm_provider),
+                model=model,
+            )
+
+            if provider_embedding_config is None:
+                raise ValueError(f"No provider config found for model: {model}")
+
+            litellm_embedding_response: Final[EmbeddingResponse] = (
+                provider_embedding_config.transform_embedding_response(
+                    model=model,
+                    raw_response=httpx_response,
+                    model_response=EmbeddingResponse(),
+                    logging_obj=logging_obj,
+                    optional_params={},  # mutable-ok: empty dict kwarg for transform_embedding_response
+                    api_key="",
+                    request_data=dict(request_data),  # mutable-ok: transform_embedding_response wants a dict
+                    litellm_params={},  # mutable-ok: empty dict kwarg for transform_embedding_response
+                )
+            )
+
+            return litellm_embedding_response
+
+        return None
+
+    def handle_logging_collected_chunks(
+        self,
+        all_chunks: Sequence[str],
+        litellm_logging_obj: LiteLLMLoggingObj,
+        model: str,
+        custom_llm_provider: str,
+        endpoint: str,
+    ) -> CostResponseTypes | None:
+        """
+        1. Convert all_chunks to a ModelResponseStream
+        2. combine model_response_stream to model_response
+        3. Return the model_response
+        """
+
+        from litellm.litellm_core_utils.streaming_handler import (
+            convert_generic_chunk_to_model_response_stream,
+            generic_chunk_has_all_required_fields,
+        )
+        from litellm.main import stream_chunk_builder
+        from litellm.types.utils import ModelResponseStream
+
+        all_translated_chunks: Final[list[object]] = []  # mutable-ok: accumulator
+
+        for chunk in all_chunks:
+            chunk = chunk.strip()
+            if not chunk or chunk == "[DONE]":
+                continue
+            chunk = chunk.removeprefix("data: ")
+            try:
+                message = json.loads(chunk)
+            except json.JSONDecodeError:
+                continue
+
+            gigachat_iterator = GigaChatModelResponseIterator(
+                streaming_response=None,
+                sync_stream=False,
+            )
+            translated_chunk = gigachat_iterator.chunk_parser(chunk=message)
+
+            if isinstance(translated_chunk, dict) and generic_chunk_has_all_required_fields(  # pyright: ignore[reportUnnecessaryIsInstance]  # runtime guard for patched chunk_parser
+                dict(translated_chunk)
+            ):
+                chunk_obj = convert_generic_chunk_to_model_response_stream(
+                    translated_chunk  # pyright: ignore[reportArgumentType]  # validated TypedDict
+                )
+            elif isinstance(translated_chunk, ModelResponseStream):
+                chunk_obj = translated_chunk
+            else:
+                continue
+
+            all_translated_chunks.append(chunk_obj)
+
+        if len(all_translated_chunks) > 0:
+            return stream_chunk_builder(
+                chunks=all_translated_chunks,
+                logging_obj=litellm_logging_obj,
+            )
+        return None
+
+    @staticmethod
+    def get_api_base(api_base: str | None = None) -> str | None:
+        return api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL
+
+    @staticmethod
+    def get_api_key(
+        api_key: str | None = None,
+    ) -> str | None:
+        return api_key or get_secret_str("GIGACHAT_API_KEY")
+
+    @staticmethod
+    def get_base_model(model: str) -> str | None:
+        return model
+
+    def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
+        return list(super().get_models(api_key, api_base))

litellm/llms/gigachat/utils.py

@@ -0,0 +1,26 @@
+from collections.abc import Mapping
+from typing import Final
+
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.utils import PromptTokensDetailsWrapper, Usage
+
+# GigaChat API endpoint
+GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1"
+
+
+def convert_usage(usage_data: Mapping[str, int]) -> Usage:
+    precached_prompt_tokens: Final = usage_data.get("precached_prompt_tokens", 0)
+    prompt_tokens_details: Final = (
+        PromptTokensDetailsWrapper(cached_tokens=precached_prompt_tokens) if precached_prompt_tokens > 0 else None
+    )
+
+    return Usage(
+        prompt_tokens=usage_data.get("prompt_tokens", 0) + precached_prompt_tokens,
+        completion_tokens=usage_data.get("completion_tokens", 0),
+        prompt_tokens_details=prompt_tokens_details,
+        total_tokens=usage_data.get("total_tokens", 0) + precached_prompt_tokens,
+    )
+
+
+def get_api_base(api_base: str | None = None) -> str | None:
+    return api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL

litellm/llms/oci/chat/cohere.py

@@ -9,7 +9,7 @@ response parsing, and streaming chunk parsing for models served with
 import datetime
 import json
 from collections.abc import Iterable, Mapping, Sequence
-from typing import Any, Final
+from typing import Final
 
 import httpx
 from pydantic import JsonValue, TypeAdapter, ValidationError
@@ -76,7 +76,7 @@ def _content_text(content: str | Iterable[Mapping[str, object]] | None) -> str:
     return str(content)
 
 
-def _extract_text_content(content: Any) -> str:
+def _extract_text_content(content: str | Iterable[Mapping[str, object]] | None) -> str:
     """Return the plain-text representation of a message content value."""
     return _content_text(content)
 

litellm/llms/runwayml/videos/transformation.py

@@ -160,14 +160,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
             **self._prompt_image_param(video_create_optional_params),
             **self._ratio_param(video_create_optional_params),
             **self._duration_param(video_create_optional_params),
-            # Pass through other parameters that aren't OpenAI-specific
             **{key: value for key, value in video_create_optional_params.items() if key not in supported_openai_params},
         }
 
     @staticmethod
     def _prompt_image_param(video_create_optional_params: VideoCreateOptionalRequestParams) -> Mapping[str, object]:
         # Handle input_reference parameter - map to promptImage
-        # RunwayML supports URLs and data URIs directly
         if "input_reference" in video_create_optional_params:
             return {"promptImage": video_create_optional_params["input_reference"]}
         return {}

litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py

@@ -182,7 +182,6 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
             else None
         )
 
-        # Generation config with proper structure for image editing
         generation_config: Final[dict[str, object]] = {
             key: value for key, value in (("response_modalities", ["IMAGE"]), ("image_config", image_config)) if value
         }

litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py

@@ -203,7 +203,6 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
             if value is not None
         }
 
-        # Build the request body for Vertex AI RAG API
         query_body: Final[Mapping[str, object]] = {
             key: value
             for key, value in (("text", query), ("rag_retrieval_config", rag_retrieval_config or None))
@@ -294,7 +293,6 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
         # Add metadata if provided
         metadata: Final = vector_store_create_optional_params.get("metadata")
 
-        # Build the request body for Vertex AI RAG Corpus creation
         request_body: Final[dict[str, object]] = {
             key: value
             for key, value in (

litellm/main.py

@@ -5507,6 +5507,9 @@ def completion(
             tpm=kwargs.get("tpm"),
             rpm=kwargs.get("rpm"),
             use_xai_oauth=kwargs.get("use_xai_oauth", False),
+            gigachat_scope=kwargs.get("gigachat_scope"),
+            gigachat_auth_url=kwargs.get("gigachat_auth_url"),
+            gigachat_access_token=kwargs.get("gigachat_access_token"),
             **{key: kwargs[key] for key in FORWARDED_KWARGS_KEYS if key in kwargs},
         )
         cast(LiteLLMLoggingObj, logging).update_environment_variables(

litellm/model_prices_and_context_window_backup.json

@@ -553,6 +553,26 @@
         "supports_response_schema": true,
         "supports_vision": true
     },
+    "amazon.nova-sonic-v1:0": {
+        "input_cost_per_audio_token": 3.4e-06,
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "bedrock",
+        "mode": "realtime",
+        "output_cost_per_audio_token": 1.36e-05,
+        "output_cost_per_token": 2.4e-07,
+        "supports_audio_input": true,
+        "supports_audio_output": true
+    },
+    "amazon.nova-2-sonic-v1:0": {
+        "input_cost_per_audio_token": 3e-06,
+        "input_cost_per_token": 3.3e-07,
+        "litellm_provider": "bedrock",
+        "mode": "realtime",
+        "output_cost_per_audio_token": 1.2e-05,
+        "output_cost_per_token": 2.75e-06,
+        "supports_audio_input": true,
+        "supports_audio_output": true
+    },
     "amazon.rerank-v1:0": {
         "input_cost_per_query": 0.001,
         "input_cost_per_token": 0.0,
@@ -19566,6 +19586,61 @@
         "supports_system_messages": true,
         "supports_tool_choice": true
     },
+    "friendliai/zai-org/GLM-5.3-Flash": {
+        "litellm_provider": "friendliai",
+        "supports_reasoning": true,
+        "supports_function_calling": true,
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 5e-07,
+        "cache_read_input_token_cost": 3e-08,
+        "supports_prompt_caching": true,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_native_structured_output": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "mode": "chat",
+        "comment": "Native multimodal GLM model for efficient coding and long-horizon agent tasks",
+        "source": "https://api.friendli.ai/serverless/v1/models",
+        "supports_vision": true,
+        "supports_image_input": true,
+        "supports_video_input": true
+    },
+    "friendliai/zai-org/GLM-5.3": {
+        "litellm_provider": "friendliai",
+        "supports_reasoning": true,
+        "supports_function_calling": true,
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "input_cost_per_token": 1.26e-06,
+        "output_cost_per_token": 3.96e-06,
+        "cache_read_input_token_cost": 2.34e-07,
+        "supports_prompt_caching": true,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_native_structured_output": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "mode": "chat",
+        "comment": "Flagship GLM model for long-horizon coding, agents, and complex project delivery",
+        "source": "https://api.friendli.ai/serverless/v1/models",
+        "supports_vision": false,
+        "supports_image_input": false
+    },
     "ft:babbage-002": {
         "deprecation_date": "2026-10-23",
         "input_cost_per_token": 1.6e-06,
@@ -24344,7 +24419,7 @@
         "supports_response_schema": true,
         "supports_vision": true
     },
-    "gigachat/GigaChat-2-Lite": {
+    "gigachat/GigaChat-2": {
         "input_cost_per_token": 0.0,
         "litellm_provider": "gigachat",
         "max_input_tokens": 128000,
@@ -24406,6 +24481,15 @@
         "output_cost_per_token": 0.0,
         "output_vector_size": 2560
     },
+    "gigachat/GigaEmbeddings-3B-2025-09": {
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "gigachat",
+        "max_input_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "embedding",
+        "output_cost_per_token": 0.0,
+        "output_vector_size": 2048
+    },
     "gmi/anthropic/claude-opus-4.5": {
         "input_cost_per_token": 5e-06,
         "litellm_provider": "gmi",

litellm/models/model.py

@@ -29,6 +29,8 @@ class LiteLLM_ProxyModelTable(LiteLLMPydanticObjectBase):
     @model_validator(mode="before")
     @classmethod
     def check_potential_json_str(cls, values):
+        if not isinstance(values, dict):
+            return values
         if isinstance(values.get("litellm_params"), str):
             try:
                 values["litellm_params"] = json.loads(values["litellm_params"])

litellm/passthrough/main.py

@@ -2,17 +2,22 @@
 This module is used to pass through requests to the LLM APIs.
 """
 
+from __future__ import annotations
+
 import asyncio
 import contextvars
-from collections.abc import AsyncGenerator, Coroutine, Generator
+from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Generator, Iterator
 from functools import partial
-from typing import TYPE_CHECKING, Any, Final, Optional, cast
+from types import TracebackType
+from typing import Any, Final, cast
 
 import httpx
-from httpx._types import CookieTypes, QueryParamTypes, RequestFiles
+from httpx._types import CookieTypes, QueryParamTypes, RequestContent, RequestFiles
 
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from litellm.passthrough.utils import CommonUtils
@@ -21,9 +26,222 @@ from litellm.utils import client
 base_llm_http_handler = BaseLLMHTTPHandler()
 from .utils import BasePassthroughUtils
 
-if TYPE_CHECKING:
-    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-    from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
+
+async def _as_async_generator(iterable: AsyncIterator[bytes]) -> AsyncGenerator[bytes, Any]:
+    async for chunk in iterable:
+        yield chunk
+
+
+def _as_generator(iterable: Iterator[bytes]) -> Generator[bytes, Any, Any]:
+    yield from iterable
+
+
+class AsyncPassthroughStreamingResponse(AsyncGenerator[Any, Any]):
+    def __init__(
+        self,
+        response: Coroutine[Any, Any, httpx.Response],
+        litellm_logging_obj: LiteLLMLoggingObj,
+        provider_config: BasePassthroughConfig,
+    ) -> None:
+        self._initialized = False
+        self._status_code: int = 0
+        self._headers = httpx.Headers()
+        self._response_coro = response
+        self._response: httpx.Response
+        self._iterator: AsyncGenerator[bytes, Any]
+        self._litellm_logging_obj = litellm_logging_obj
+        self._provider_config = provider_config
+        self._raw_bytes: list[bytes] = []  # mutable-ok: instance buffer for streaming chunks
+        self._flush_scheduled = False
+        self._background_tasks: set[asyncio.Task] = set()  # mutable-ok: instance set for background task tracking
+        self._hidden_params: dict[str, object] = {}  # mutable-ok: router attaches response headers here in place
+
+    @property
+    def status_code(self) -> int:
+        if not self._initialized:
+            raise RuntimeError("AsyncPassthroughStreamingResponse must be awaited before accessing status_code")
+        return self._status_code
+
+    @status_code.setter
+    def status_code(self, value: int) -> None:
+        self._status_code = value
+
+    @property
+    def headers(self) -> httpx.Headers:
+        if not self._initialized:
+            raise RuntimeError("AsyncPassthroughStreamingResponse must be awaited before accessing headers")
+        return self._headers
+
+    @headers.setter
+    def headers(self, value: httpx.Headers) -> None:
+        self._headers = value
+
+    def __await__(self) -> Iterator[Any]:
+        async def _init():
+            if not self._initialized:
+                self._response = await self._response_coro
+                self.headers = self._response.headers
+                self.status_code = self._response.status_code
+                self._initialized = True
+                try:
+                    self._response.raise_for_status()
+                    self._iterator = _as_async_generator(self._response.aiter_bytes())
+                except Exception:  # noqa: BLE001 # Safe catch-all for cleanup logic
+                    try:
+                        await self._response.aread()
+                    except Exception:  # noqa: BLE001 S110 # Safe catch-all for cleanup logic
+                        pass
+                    try:
+                        await self._response.aclose()
+                    except Exception:  # noqa: BLE001 S110 # Safe catch-all for cleanup logic
+                        pass
+                    raise
+            return self
+
+        return _init().__await__()
+
+    def _start_flush(self) -> None:
+        if self._flush_scheduled or not self._raw_bytes:
+            return
+        self._flush_scheduled = True
+
+        try:
+            task: Final = asyncio.create_task(
+                self._litellm_logging_obj.async_flush_passthrough_collected_chunks(
+                    raw_bytes=self._raw_bytes,
+                    provider_config=self._provider_config,
+                )
+            )
+
+            # Compliant: Save a strong reference to prevent GC
+            self._background_tasks.add(task)
+
+            # Remove the task from the set when it finishes to avoid memory leaks
+            task.add_done_callback(self._background_tasks.discard)
+        except Exception as e:  # noqa: BLE001 # Safe catch-all for verbose logging
+            verbose_logger.exception(
+                "Failed to schedule passthrough spend-tracking flush; %d buffered chunks dropped: %s",
+                len(self._raw_bytes),
+                e,
+            )
+
+    def __aiter__(self) -> AsyncPassthroughStreamingResponse:
+        return self
+
+    def aiter_bytes(self) -> AsyncPassthroughStreamingResponse:
+        return self
+
+    async def __anext__(self) -> bytes:
+        if not self._initialized:
+            await self  # pyright: ignore[reportGeneralTypeIssues]  # structural type check misses __await__
+        try:
+            chunk: Final = await anext(self._iterator)
+            self._raw_bytes.append(chunk)
+        except Exception:  # noqa: BLE001 # Safe catch-all for cleanup logic
+            self._start_flush()
+            try:
+                await self._response.aclose()
+            except Exception:  # noqa: BLE001 S110 # Safe catch-all for cleanup logic
+                pass
+            raise
+        else:
+            return chunk
+
+    async def asend(self, value: bytes) -> bytes:
+        if not self._initialized:
+            await self  # pyright: ignore[reportGeneralTypeIssues]  # structural type check misses __await__
+        return await self._iterator.asend(value)
+
+    async def athrow(
+        self,
+        typ: BaseException | type[BaseException],
+        val: BaseException | object = None,
+        tb: TracebackType | None = None,
+    ) -> bytes:
+        if not self._initialized:
+            await self  # pyright: ignore[reportGeneralTypeIssues]  # structural type check misses __await__
+        return await self._iterator.athrow(typ, val, tb)  # pyright: ignore[reportCallIssue, reportArgumentType]  # matches one of the athrow overloads
+
+    async def aclose(self) -> None:
+        self._start_flush()
+        try:
+            if self._initialized:
+                await self._iterator.aclose()
+                await self._response.aclose()
+        except Exception:  # noqa: BLE001 S110 # Safe catch-all for cleanup logic
+            pass
+
+
+class PassthroughStreamingResponse(Generator[Any, Any, Any]):
+    def __init__(
+        self,
+        response: httpx.Response,
+        litellm_logging_obj: LiteLLMLoggingObj,
+        provider_config: BasePassthroughConfig,
+    ) -> None:
+        self._response = response
+        self.headers = response.headers
+        self.status_code = response.status_code
+        self._litellm_logging_obj = litellm_logging_obj
+        self._provider_config = provider_config
+        self._iterator: Generator[bytes, Any, Any] = _as_generator(response.iter_bytes())
+        self._raw_bytes: list[bytes] = []  # mutable-ok: instance buffer for streaming chunks
+        self._flush_scheduled = False
+
+    def _start_flush(self) -> None:
+        if self._flush_scheduled or not self._raw_bytes:
+            return
+        self._flush_scheduled = True
+
+        from litellm.utils import executor
+
+        try:
+            executor.submit(
+                self._litellm_logging_obj.flush_passthrough_collected_chunks,
+                raw_bytes=self._raw_bytes,
+                provider_config=self._provider_config,
+            )
+        except Exception as e:  # noqa: BLE001 # Safe catch-all for verbose logging
+            verbose_logger.exception(
+                "Failed to schedule passthrough spend-tracking flush; %d buffered chunks dropped: %s",
+                len(self._raw_bytes),
+                e,
+            )
+
+    def __iter__(self) -> PassthroughStreamingResponse:
+        return self
+
+    def __next__(self) -> bytes:
+        try:
+            chunk: Final = next(self._iterator)
+            self._raw_bytes.append(chunk)
+        except Exception:  # noqa: BLE001 # Safe catch-all for cleanup logic
+            self._start_flush()
+            try:
+                self._response.close()
+            except Exception:  # noqa: BLE001 S110 # Safe catch-all for cleanup logic
+                pass
+            raise
+        else:
+            return chunk
+
+    def send(self, value: bytes) -> bytes:
+        return self._iterator.send(value)
+
+    def throw(
+        self,
+        typ: BaseException | type[BaseException],
+        val: BaseException | object = None,
+        tb: TracebackType | None = None,
+    ) -> bytes:
+        return self._iterator.throw(typ, val, tb)  # pyright: ignore[reportCallIssue, reportArgumentType]  # matches one of the throw overloads
+
+    def close(self) -> None:
+        self._start_flush()
+        try:
+            self._response.close()
+        except Exception:  # noqa: BLE001 S110 # Safe catch-all for cleanup logic
+            pass
 
 
 @client
@@ -37,10 +255,10 @@ async def allm_passthrough_route(
     api_key: str | None = None,
     request_query_params: dict | None = None,
     request_headers: dict | None = None,
-    content: Any | None = None,
+    content: RequestContent | None = None,
     data: dict | None = None,
     files: RequestFiles | None = None,
-    json: Any | None = None,
+    json: object | None = None,
     params: QueryParamTypes | None = None,
     cookies: CookieTypes | None = None,
     client: HTTPHandler | AsyncHTTPHandler | None = None,
@@ -64,7 +282,7 @@ async def allm_passthrough_route(
         from litellm.utils import ProviderConfigManager
 
         provider_config = cast(
-            Optional["BasePassthroughConfig"], kwargs.get("provider_config")
+            BasePassthroughConfig | None, kwargs.get("provider_config")
         ) or ProviderConfigManager.get_provider_passthrough_config(
             provider=LlmProviders(custom_llm_provider),
             model=model,
@@ -132,12 +350,12 @@ async def allm_passthrough_route(
         if resolved_custom_llm_provider:
             try:
                 provider_config = cast(
-                    Optional["BasePassthroughConfig"], kwargs.get("provider_config")
+                    BasePassthroughConfig | None, kwargs.get("provider_config")
                 ) or ProviderConfigManager.get_provider_passthrough_config(
                     provider=LlmProviders(resolved_custom_llm_provider),
                     model=model,
                 )
-            except Exception:
+            except Exception:  # noqa: BLE001 S110
                 # If we can't get provider config, pass None
                 pass
 
@@ -162,10 +380,10 @@ def llm_passthrough_route(
     api_key: str | None = None,
     request_query_params: dict | None = None,
     request_headers: dict | None = None,
-    content: Any | None = None,
+    content: RequestContent | None = None,
     data: dict | None = None,
     files: RequestFiles | None = None,
-    json: Any | None = None,
+    json: object | None = None,
     params: QueryParamTypes | None = None,
     cookies: CookieTypes | None = None,
     client: HTTPHandler | AsyncHTTPHandler | None = None,
@@ -190,7 +408,9 @@ def llm_passthrough_route(
 
     _is_async: Final = bool(kwargs.get("allm_passthrough_route", False))
 
-    litellm_logging_obj: Final = cast("LiteLLMLoggingObj", kwargs.get("litellm_logging_obj"))
+    litellm_logging_obj: Final = cast(
+        LiteLLMLoggingObj, kwargs.get("litellm_logging_obj")
+    )  # cast-ok: logging obj is constructed upstream; tests inject mocks
 
     model, custom_llm_provider, api_key, api_base = get_llm_provider(
         model=model,
@@ -235,7 +455,7 @@ def llm_passthrough_route(
     )
 
     provider_config: Final = cast(
-        Optional["BasePassthroughConfig"], kwargs.get("provider_config")
+        BasePassthroughConfig | None, kwargs.get("provider_config")
     ) or ProviderConfigManager.get_provider_passthrough_config(
         provider=LlmProviders(custom_llm_provider),
         model=model,
@@ -276,10 +496,13 @@ def llm_passthrough_route(
         forward_headers=False,
     )
 
+    _request_data: dict | None = (
+        data if isinstance(data, dict) else (json if isinstance(json, dict) else None)
+    )  # rebind-ok: conditional
     headers, signed_json_body = provider_config.sign_request(
         headers=headers,
         litellm_params=litellm_params_dict,
-        request_data=data if data else json,
+        request_data=_request_data,
         api_base=str(updated_url),
         model=model,
     )
@@ -301,9 +524,12 @@ def llm_passthrough_route(
     )
 
     ## IS STREAMING REQUEST
+    _streaming_request_data: dict = (
+        data if isinstance(data, dict) else (json if isinstance(json, dict) else {})
+    )  # rebind-ok: conditional
     is_streaming_request: Final = provider_config.is_streaming_request(
         endpoint=endpoint,
-        request_data=data or json or {},
+        request_data=_streaming_request_data,
     )
 
     # Update logging object with streaming status
@@ -334,18 +560,26 @@ def llm_passthrough_route(
         else:
             # Sync path - client.client.send returns Response directly
             response: httpx.Response = client.client.send(request=request, stream=is_streaming_request)
-            response.raise_for_status()
+            try:
+                response.raise_for_status()
+            except Exception:  # noqa: BLE001 # Safe catch-all for cleanup logic
+                try:
+                    response.read()
+                except Exception:  # noqa: BLE001 S110 # Safe catch-all for cleanup logic
+                    pass
+                try:
+                    response.close()
+                except Exception:  # noqa: BLE001 S110 # Safe catch-all for cleanup logic
+                    pass
+                raise
 
-            if (
-                hasattr(response, "iter_bytes") and is_streaming_request
-            ):  # yield the chunk, so we can store it in the logging object
-                return _sync_streaming(response, litellm_logging_obj, provider_config)
+            if hasattr(response, "iter_bytes") and is_streaming_request:
+                return PassthroughStreamingResponse(response, litellm_logging_obj, provider_config)
             else:
-                # For non-streaming responses, yield the entire response
                 return response
     except Exception as e:
-        if provider_config is None:
-            raise e
+        # provider_config is guaranteed non-None here due to the earlier guard
+        assert provider_config is not None
         raise base_llm_http_handler._handle_error(
             e=e,
             provider_config=provider_config,
@@ -356,8 +590,8 @@ async def _async_passthrough_request(
     client: HTTPHandler | AsyncHTTPHandler,
     request: httpx.Request,
     is_streaming_request: bool,
-    litellm_logging_obj: "LiteLLMLoggingObj",
-    provider_config: "BasePassthroughConfig",
+    litellm_logging_obj: LiteLLMLoggingObj,
+    provider_config: BasePassthroughConfig,
 ) -> httpx.Response | AsyncGenerator[Any, Any]:
     """
     Handle async passthrough requests.
@@ -369,8 +603,7 @@ async def _async_passthrough_request(
     # Check if it's a coroutine and await it
     if asyncio.iscoroutine(response_result):
         if is_streaming_request:
-            # Pass the coroutine to _async_streaming which will await it
-            return _async_streaming(
+            return await AsyncPassthroughStreamingResponse(  # pyright: ignore[reportGeneralTypeIssues]  # structural type check misses __await__
                 response=response_result,
                 litellm_logging_obj=litellm_logging_obj,
                 provider_config=provider_config,
@@ -383,84 +616,3 @@ async def _async_passthrough_request(
     else:
         # Fallback for sync-like behavior (shouldn't happen in async path)
         raise Exception("Expected coroutine from async client")
-
-
-def _sync_streaming(
-    response: httpx.Response,
-    litellm_logging_obj: "LiteLLMLoggingObj",
-    provider_config: "BasePassthroughConfig",
-):
-    from litellm.utils import executor
-
-    raw_bytes: Final[list[bytes]] = []
-    flush_scheduled = False
-    try:
-        for chunk in response.iter_bytes():
-            raw_bytes.append(chunk)
-            yield chunk
-    finally:
-        if not flush_scheduled and raw_bytes:
-            flush_scheduled = True
-            try:
-                executor.submit(
-                    litellm_logging_obj.flush_passthrough_collected_chunks,
-                    raw_bytes=raw_bytes,
-                    provider_config=provider_config,
-                )
-            except Exception as e:
-                verbose_logger.exception(
-                    "Failed to schedule passthrough spend-tracking flush "
-                    "in _sync_streaming; %d buffered chunks dropped: %s",
-                    len(raw_bytes),
-                    e,
-                )
-
-
-async def _async_streaming(
-    response: Coroutine[Any, Any, httpx.Response],
-    litellm_logging_obj: "LiteLLMLoggingObj",
-    provider_config: "BasePassthroughConfig",
-):
-    iter_response: Final = await response
-
-    try:
-        iter_response.raise_for_status()
-    except Exception:
-        try:
-            await iter_response.aclose()
-        except Exception:
-            pass
-        raise
-
-    raw_bytes: Final[list[bytes]] = []
-    flush_scheduled = False
-    try:
-        async for chunk in iter_response.aiter_bytes():
-            raw_bytes.append(chunk)
-            yield chunk
-    except Exception:
-        try:
-            await iter_response.aclose()
-        except Exception:
-            pass
-        raise
-    finally:
-        # GeneratorExit (raised on client disconnect) is not caught by
-        # `except Exception`; the finally block ensures partial usage
-        # still gets flushed for spend tracking. See LIT-2642.
-        if not flush_scheduled and raw_bytes:
-            flush_scheduled = True
-            try:
-                asyncio.create_task(
-                    litellm_logging_obj.async_flush_passthrough_collected_chunks(
-                        raw_bytes=raw_bytes,
-                        provider_config=provider_config,
-                    )
-                )
-            except Exception as e:
-                verbose_logger.exception(
-                    "Failed to schedule passthrough spend-tracking flush "
-                    "in _async_streaming; %d buffered chunks dropped: %s",
-                    len(raw_bytes),
-                    e,
-                )

litellm/proxy/_types.py

@@ -474,6 +474,7 @@ class LiteLLMRoutes(enum.Enum):
         "/vllm",
         "/mistral",
         "/milvus",
+        "/gigachat",
         "/watsonx",
     ]
 

litellm/proxy/common_request_processing.py

@@ -1495,6 +1495,35 @@ class ProxyBaseLLMRequestProcessing:
     def __init__(self, data: dict):
         self.data = data
 
+    @staticmethod
+    def _merge_passthrough_streaming_headers(
+        response_headers: httpx.Headers | dict | None,
+        custom_headers: dict,
+    ) -> dict:
+        """
+        Merge upstream passthrough headers with proxy/custom headers.
+
+        Proxy/custom headers win on key collisions.
+        """
+        excluded_headers: Final = {  # mutable-ok: set of header names to exclude from forwarding
+            "transfer-encoding",
+            "content-encoding",
+            "set-cookie",
+            "connection",
+            "keep-alive",
+            "proxy-authenticate",
+            "proxy-authorization",
+            "te",
+            "trailer",
+            "upgrade",
+        }
+
+        merged_headers: Final = {  # mutable-ok: dict comprehension for merged headers forwarded to httpx
+            key: value for key, value in dict(response_headers or {}).items() if key.lower() not in excluded_headers
+        }
+        merged_headers.update(custom_headers)
+        return merged_headers
+
     @staticmethod
     def get_custom_headers(
         *,
@@ -2389,6 +2418,16 @@ class ProxyBaseLLMRequestProcessing:
                     )
 
                 if route_type == "allm_passthrough_route":
+                    upstream_response_headers: Final = getattr(response, "headers", None)
+                    streaming_headers: Final = (
+                        ProxyBaseLLMRequestProcessing._merge_passthrough_streaming_headers(
+                            response_headers=upstream_response_headers,
+                            custom_headers=custom_headers,
+                        )
+                        if upstream_response_headers is not None
+                        else custom_headers
+                    )
+
                     # Check if response is an async generator
                     if self._is_streaming_response(response):
                         if asyncio.iscoroutine(response):
@@ -2418,11 +2457,11 @@ class ProxyBaseLLMRequestProcessing:
 
                         # For passthrough routes, stream directly without error parsing
                         # since we're dealing with raw binary data (e.g., AWS event streams)
-                        return StreamingResponse(
-                            content=generator,
-                            status_code=status.HTTP_200_OK,
+                        return _UpstreamClosingStreamingResponse(
+                            content=generator,  # pyright: ignore[reportArgumentType]  # generator-configured StreamingResponse
+                            status_code=getattr(response, "status_code", status.HTTP_200_OK),
                             media_type=self._passthrough_event_stream_media_type(),
-                            headers=custom_headers,
+                            headers=streaming_headers,
                         )
                     else:
                         _early = await self._handle_non_streaming_allm_passthrough_route(
@@ -2437,7 +2476,7 @@ class ProxyBaseLLMRequestProcessing:
                         return StreamingResponse(
                             content=response.aiter_bytes(),
                             status_code=response.status_code,
-                            headers=custom_headers,
+                            headers=streaming_headers,
                         )
                 elif route_type == "anthropic_messages":
                     # Check if response is actually a streaming response (async generator)

litellm/proxy/db/budget_window_spend_writer.py

@@ -7,20 +7,15 @@ instead of aggregating LiteLLM_SpendLogs every time a window counter goes cold
 (issue #35766). Raw SQL rather than the Prisma upsert helper because the
 conditional roll cannot be expressed through the query builder.
 
-Seeding a row that does not exist yet reads LiteLLM_SpendLogs once, excluding
-the requests whose increments are in the same batch so neither source counts
-them twice. One gap survives that exclusion: without the Redis transaction
-buffer every pod flushes its own increments, so a row seeded by one pod can
-include spend logs whose increments are still queued on another pod, and those
-increments are added again when that pod flushes. That is bounded by a single
-flush interval, happens at most once per window row, and only ever over-counts:
-the seed never omits spend, because every increment not yet in the row still
-reaches it on its own pod's next flush. A row therefore lags real spend by at
-most one flush interval of queued increments, the same lag the SpendLogs
-aggregate it replaces (and every other spend column) already has.
+Seeding a row that does not exist yet reads LiteLLM_SpendLogs once and takes
+off what the increments being flushed will add, so neither source counts the
+same request twice. A row therefore lags real spend by at most one flush
+interval of increments queued elsewhere: the same lag the SpendLogs aggregate
+it replaces (and every other spend column) already has.
 """
 
 from collections.abc import Sequence
+from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Final, Protocol
 
@@ -67,33 +62,46 @@ _ROLL_WINDOW_SPEND_SQL: Final = (
 )
 
 _SEED_FROM_SPEND_LOGS_KEY_SQL: Final = (
-    'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" '
-    "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC') "
-    "AND NOT (request_id = ANY($3::text[]) AND \"startTime\" >= ($4::timestamptz AT TIME ZONE 'UTC'))"
+    "SELECT COALESCE(SUM(spend), 0.0) AS total, "
+    "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch "
+    'FROM "LiteLLM_SpendLogs" '
+    "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')"
 )
 
 _SEED_FROM_SPEND_LOGS_TEAM_SQL: Final = (
-    'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" '
-    "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC') "
-    "AND NOT (request_id = ANY($3::text[]) AND \"startTime\" >= ($4::timestamptz AT TIME ZONE 'UTC'))"
+    "SELECT COALESCE(SUM(spend), 0.0) AS total, "
+    "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch "
+    'FROM "LiteLLM_SpendLogs" '
+    "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')"
 )
 
 _SEED_FROM_SPEND_LOGS_KEY_UNBOUNDED_SQL: Final = (
-    'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" '
+    "SELECT COALESCE(SUM(spend), 0.0) AS total, COALESCE(SUM(spend), 0.0) AS before_batch "
+    'FROM "LiteLLM_SpendLogs" '
     "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')"
 )
 
 _SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL: Final = (
-    'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" '
+    "SELECT COALESCE(SUM(spend), 0.0) AS total, COALESCE(SUM(spend), 0.0) AS before_batch "
+    'FROM "LiteLLM_SpendLogs" '
     "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')"
 )
 
 _UPSERT_TRANSACTION_TIMEOUT: Final = timedelta(seconds=60)
 
 
+@dataclass(frozen=True, slots=True)
+class WindowSeedTotals:
+    """The two sums a seed needs: everything persisted for the window, and the
+    part of it that predates the batch being flushed."""
+
+    total: float
+    before_batch: float
+
+
 class WindowSpendLogsAggregate(Protocol):
-    """Sums LiteLLM_SpendLogs for one entity since window_start, ignoring the
-    requests whose ids are handed in.
+    """Sums LiteLLM_SpendLogs for one entity since window_start, split at the
+    batch's earliest request.
 
     Injected so the flush can be exercised without a database and so the
     expensive aggregate stays swappable.
@@ -105,21 +113,19 @@ class WindowSpendLogsAggregate(Protocol):
         entity_type: str,
         entity_id: str,
         window_start: datetime,
-        exclude_request_ids: Sequence[str],
-        exclude_started_at: datetime | None,
-    ) -> float | None: ...
+        batch_started_at: datetime | None,
+    ) -> WindowSeedTotals | None: ...
 
 
-async def spend_logs_total_excluding(
+async def spend_logs_seed_totals(
     prisma_client: "PrismaClient",
     entity_type: str,
     entity_id: str,
     window_start: datetime,
-    exclude_request_ids: Sequence[str],
-    exclude_started_at: datetime | None,
-) -> float | None:
-    """LiteLLM_SpendLogs spend for one entity since window_start, minus the
-    requests already accounted for by the increments being flushed.
+    batch_started_at: datetime | None,
+) -> WindowSeedTotals | None:
+    """LiteLLM_SpendLogs spend for one entity since window_start, both in full
+    and up to the start of the batch being flushed, in one scan.
 
     The spend log writer drains its own queue on a ~2s poll whenever anything
     is queued, while window increments flush on the much slower batch tick, so
@@ -127,13 +133,12 @@ async def spend_logs_total_excluding(
     already in the table. Counting them in the seed and again in the increment
     is what made a fresh row land at twice the true spend.
 
-    The exclusion is bounded to rows that started at or after the batch's
-    earliest request. request_id can be chosen by the client
-    (x-litellm-call-id), so an unbounded exclusion would let a replayed old id
-    erase a historical row from the seed while its increment still lands.
-    Without a known start the batch's ids are not excluded at all: that can
-    only over-count once, which enforcement tolerates, whereas under-counting
-    is a budget bypass.
+    Both halves are needed because neither is safe alone: the full sum
+    double-counts this batch, and the sum before the batch drops spend another
+    pod has already persisted but not yet incremented. _seed_base picks between
+    them. Without a known batch start the two are the same sum, so the seed
+    counts everything: that can only over-count once, which enforcement
+    tolerates, whereas under-counting is a budget bypass.
     """
     if entity_type == Litellm_EntityType.KEY.value:
         bounded_sql, unbounded_sql = _SEED_FROM_SPEND_LOGS_KEY_SQL, _SEED_FROM_SPEND_LOGS_KEY_UNBOUNDED_SQL
@@ -143,21 +148,23 @@ async def spend_logs_total_excluding(
         return None
     rows: Final = (
         await prisma_client.db.query_raw(unbounded_sql, entity_id, window_start)
-        if exclude_started_at is None or not exclude_request_ids
+        if batch_started_at is None
         else await prisma_client.db.query_raw(
             bounded_sql,
             entity_id,
             window_start,
-            tuple(exclude_request_ids),
-            _exclusion_lower_bound(exclude_started_at),
+            _exclusion_upper_bound(batch_started_at),
         )
     )
     if not rows:
-        return 0.0
-    return float(rows[0].get("total") or 0.0)
+        return WindowSeedTotals(total=0.0, before_batch=0.0)
+    return WindowSeedTotals(
+        total=float(rows[0].get("total") or 0.0),
+        before_batch=float(rows[0].get("before_batch") or 0.0),
+    )
 
 
-def _exclusion_lower_bound(started_at: datetime) -> datetime:
+def _exclusion_upper_bound(started_at: datetime) -> datetime:
     """LiteLLM_SpendLogs.startTime is TIMESTAMP(3); floor to the second so a
     millisecond rounding of the batch's own earliest row cannot slip under it."""
     return to_naive_utc(started_at).replace(microsecond=0)
@@ -194,20 +201,33 @@ async def _seed_base_for_missing_row(
 
     This is the LiteLLM_SpendLogs aggregate the window counter reseed runs on
     every cold counter today, but here it runs once per window lifetime and off
-    the request path, and it excludes this batch's own requests so they are
-    counted by their increments alone.
+    the request path, and it discounts the queued increments so they are
+    counted once.
     """
     if _primary_key(transaction) in existing_primary_keys:
         return 0.0
-    base: Final = await spend_logs_aggregate(
+    totals: Final = await spend_logs_aggregate(
         prisma_client=prisma_client,
         entity_type=transaction["entity_type"],
         entity_id=transaction["entity_id"],
         window_start=datetime.fromisoformat(transaction["window_start"]).replace(tzinfo=timezone.utc),
-        exclude_request_ids=transaction["request_ids"],
-        exclude_started_at=_transaction_started_at(transaction),
+        batch_started_at=_transaction_started_at(transaction),
     )
-    return float(base or 0.0)
+    if totals is None:
+        return 0.0
+    return _seed_base(totals=totals, batch_spend=transaction["spend"])
+
+
+def _seed_base(totals: WindowSeedTotals, batch_spend: float) -> float:
+    """What the window already held before the increments about to be applied.
+
+    Subtracting the batch's own spend from the full sum keeps every other
+    request in the seed, including the ones another pod persisted and has not
+    incremented yet, which a plain cutoff would drop for good if that pod died.
+    When this batch's own log rows have not landed yet the subtraction takes
+    spend that was never counted, so the sum before the batch is the floor.
+    """
+    return max(totals.total - batch_spend, totals.before_batch)
 
 
 def _transaction_started_at(transaction: WindowSpendTransaction) -> datetime | None:
@@ -241,7 +261,7 @@ def _upsert_params(
 async def commit_window_spend_updates(
     prisma_client: "PrismaClient",
     transactions: Sequence[WindowSpendTransaction],
-    spend_logs_aggregate: WindowSpendLogsAggregate = spend_logs_total_excluding,
+    spend_logs_aggregate: WindowSpendLogsAggregate = spend_logs_seed_totals,
 ) -> None:
     """Apply aggregated window increments to LiteLLM_BudgetWindowSpend.
 

litellm/proxy/db/db_spend_update_writer.py

@@ -215,11 +215,7 @@ class DBSpendUpdateWriter:
         start_time: datetime | None,
         end_time: datetime | None,
         response_cost: float | None,
-    ) -> str | None:
-        """Returns the LiteLLM_SpendLogs request_id this call was recorded
-        under, so the caller can tell the budget-window writer which log rows
-        its increments already cover. None when the payload could not be built.
-        """
+    ) -> None:
         from litellm.proxy.proxy_server import (
             disable_spend_logs,
             litellm_proxy_budget_name,
@@ -236,7 +232,7 @@ class DBSpendUpdateWriter:
                 team_id,
             )
             if ProxyUpdateSpend.disable_spend_updates() is True:
-                return None
+                return
             if token is not None and isinstance(token, str) and token.startswith("sk-"):
                 hashed_token = hash_token(token=token)
             else:
@@ -310,7 +306,6 @@ class DBSpendUpdateWriter:
             )
 
             verbose_proxy_logger.debug("Runs spend update on all tables")
-            return payload.get("request_id")
         except Exception:
             spend_log_error(
                 "Spend tracking - update_database failed. Spend log insertion or daily transaction enqueue "
@@ -323,7 +318,7 @@ class DBSpendUpdateWriter:
                 org_id,
                 end_user_id,
             )
-            return None
+            return
 
     async def _enqueue_tool_usage_transaction(
         self,

litellm/proxy/db/db_transaction_queue/redis_update_buffer.py

@@ -46,6 +46,7 @@ from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdate
 from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
     WindowSpendTransaction,
     WindowSpendUpdateQueue,
+    to_wire_payload,
 )
 from litellm.secret_managers.main import str_to_bool
 from litellm.types.caching import (
@@ -298,7 +299,7 @@ class RedisUpdateBuffer:
                 ServiceTypes.REDIS_DAILY_AGENT_SPEND_UPDATE_QUEUE,
             ),
             (
-                window_spend_update_transactions,
+                tuple(map(to_wire_payload, window_spend_update_transactions)),
                 REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY,
                 ServiceTypes.REDIS_WINDOW_SPEND_UPDATE_QUEUE,
             ),
@@ -484,7 +485,12 @@ class RedisUpdateBuffer:
             (daily_end_user_spend_update_transactions, REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY),
             (daily_agent_spend_update_transactions, REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY),
             (daily_tag_spend_update_transactions, REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY),
-            (window_spend_update_transactions, REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY),
+            (
+                None
+                if window_spend_update_transactions is None
+                else tuple(map(to_wire_payload, window_spend_update_transactions)),
+                REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY,
+            ),
         )
 
         rpush_list: Final = tuple(

litellm/proxy/db/db_transaction_queue/window_spend_update_queue.py

@@ -26,17 +26,12 @@ class WindowSpendTransaction(TypedDict):
     window_start is an ISO-8601 string rather than a datetime so the
     transaction survives the JSON round trip through the Redis buffer.
 
-    request_ids carries the LiteLLM_SpendLogs ids this spend came from. The
-    one-time seed for a window that has no row yet subtracts them from its
-    LiteLLM_SpendLogs aggregate, because the spend log writer flushes on its
-    own ~2s poll and will usually have persisted these rows before the window
-    queue flushes; without the exclusion the seed and the increment would each
-    count them.
-
-    started_at is the earliest request start in the batch. The seed only
-    subtracts a request_id whose LiteLLM_SpendLogs.startTime is at or after it,
-    so a client that replays an old id through x-litellm-call-id cannot make the
-    seed drop the historical row that id already paid for.
+    started_at is the earliest request start in the batch. The one-time seed for
+    a window that has no row yet uses it to tell this batch's own
+    LiteLLM_SpendLogs rows from everything else, because the spend log writer
+    flushes on its own ~2s poll and will usually have persisted this batch's
+    rows before the window queue flushes; without that split the seed and the
+    increment would each count them.
     """
 
     entity_type: ReadOnly[str]
@@ -44,10 +39,37 @@ class WindowSpendTransaction(TypedDict):
     window_duration: ReadOnly[str]
     window_start: ReadOnly[str]
     spend: ReadOnly[float]
-    request_ids: ReadOnly[Sequence[str]]
     started_at: ReadOnly[str | None]
 
 
+class WindowSpendWirePayload(WindowSpendTransaction):
+    """How an increment is encoded in the shared Redis buffer.
+
+    request_ids is dead weight here: workers built before this field was
+    dropped index it while merging whatever they pop, and the pop is
+    destructive, so a leader still running one of those during a rolling deploy
+    would raise on a payload without the key and lose those increments. It is
+    always empty, which only makes such a leader seed without exclusions.
+
+    TODO: remove once no supported version reads it, i.e. one release after the
+    field stopped being written.
+    """
+
+    request_ids: ReadOnly[Sequence[str]]
+
+
+def to_wire_payload(transaction: WindowSpendTransaction) -> WindowSpendWirePayload:
+    return WindowSpendWirePayload(
+        entity_type=transaction["entity_type"],
+        entity_id=transaction["entity_id"],
+        window_duration=transaction["window_duration"],
+        window_start=transaction["window_start"],
+        spend=transaction["spend"],
+        started_at=transaction.get("started_at"),
+        request_ids=(),
+    )
+
+
 def to_naive_utc(value: datetime) -> datetime:
     """LiteLLM_BudgetWindowSpend.window_start is TIMESTAMP(3), which holds naive UTC."""
     if value.tzinfo is None:
@@ -72,7 +94,6 @@ def build_window_spend_transaction(
     window_duration: str,
     window_start: datetime,
     spend: float,
-    request_id: str | None = None,
     started_at: datetime | None = None,
 ) -> WindowSpendTransaction:
     return WindowSpendTransaction(
@@ -81,7 +102,6 @@ def build_window_spend_transaction(
         window_duration=window_duration,
         window_start=to_naive_utc(window_start).isoformat(timespec="microseconds"),
         spend=spend,
-        request_ids=() if request_id is None else (request_id,),
         started_at=None
         if started_at is None
         else to_naive_utc(started_at.astimezone(timezone.utc)).isoformat(timespec="microseconds"),
@@ -101,7 +121,6 @@ def _merge_window_spend_transactions(
         window_duration=first["window_duration"],
         window_start=first["window_start"],
         spend=math.fsum(payload["spend"] for payload in payloads),
-        request_ids=tuple(sorted(frozenset(chain.from_iterable(payload["request_ids"] for payload in payloads)))),
         started_at=min(started_ats) if started_ats else None,
     )
 

litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py

@@ -433,7 +433,7 @@ class HeadroomGuardrail(CustomGuardrail):
             payload["model"] = model
 
         try:
-            raw_response: HttpxResponse = await self.async_handler.post(  # pyright: ignore[reportUnknownMemberType]
+            raw_response: HttpxResponse = await self.async_handler.post(  # pyright: ignore[reportUnknownMemberType]  # AsyncHTTPHandler.post is untyped
                 url=f"{self.headroom_api_base}/v1/compress",
                 json=payload,
                 headers=self._request_headers(),
@@ -570,7 +570,7 @@ class HeadroomGuardrail(CustomGuardrail):
             params["query"] = query
 
         try:
-            raw_response: HttpxResponse = await self.async_handler.get(  # pyright: ignore[reportUnknownMemberType]
+            raw_response: HttpxResponse = await self.async_handler.get(  # pyright: ignore[reportUnknownMemberType]  # AsyncHTTPHandler.get is untyped
                 url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}",
                 params=params,
                 headers=self._request_headers(),

litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py

@@ -197,7 +197,7 @@ class RepelloAIGuardrail(CustomGuardrail):
         repelloai_response: RepelloAIAnalyzeResponse | None = None
         try:
             verbose_proxy_logger.debug("RepelloAI Argus request: %s", request)
-            response: Final[HttpxResponse] = await self.async_handler.post(  # pyright: ignore[reportUnknownMemberType]
+            response: Final[HttpxResponse] = await self.async_handler.post(  # pyright: ignore[reportUnknownMemberType]  # AsyncHTTPHandler.post is untyped
                 url=endpoint,
                 headers={"X-API-Key": self.repelloai_api_key},
                 json=request,

litellm/proxy/hooks/proxy_track_cost_callback.py

@@ -587,7 +587,7 @@ async def _update_database_and_spend_counters(
     model_access_groups: Sequence[str] | None = None,
 ) -> None:
     try:
-        spend_log_request_id = await proxy_logging_obj.db_spend_update_writer.update_database(
+        await proxy_logging_obj.db_spend_update_writer.update_database(
             token=user_api_key,
             response_cost=response_cost,
             user_id=user_id,
@@ -623,7 +623,6 @@ async def _update_database_and_spend_counters(
             budget_reservation=budget_reservation,
             end_user_id=end_user_id,
             tags=request_tags,
-            request_id=spend_log_request_id,
             request_started_at=start_time,
             model_access_groups=model_access_groups,
         )

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -7,7 +7,7 @@ POST /auto_router/validate_complexity_router_config - Dry-run the complexity-rou
 
 from collections.abc import Mapping, Sequence
 from datetime import datetime, timedelta, timezone
-from itertools import groupby
+from itertools import chain, groupby
 from operator import attrgetter
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Annotated, Final, Protocol
@@ -58,10 +58,11 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
     ComplexityRouterConfigValidationResponse,
     RequestComplexityRouterConfig,
     ShadowEvalDirection,
-    ShadowEvalJobKeyResponse,
     ShadowEvalJobResponse,
+    ShadowEvalJobTargetResponse,
     ShadowEvalResult,
     ShadowEvalSlice,
+    ShadowEvalTargetType,
     StartShadowEvalRequest,
 )
 
@@ -104,10 +105,43 @@ class _VerificationTokenTable(Protocol):
     async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_VerificationTokenRow]: ...
 
 
+class _TeamRow(Protocol):
+    @property
+    def team_id(self) -> str: ...
+
+    @property
+    def team_alias(self) -> str | None: ...
+
+
+class _TeamRowsTable(Protocol):
+    async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_TeamRow]: ...
+
+
+class _UserRow(Protocol):
+    @property
+    def user_id(self) -> str: ...
+
+    @property
+    def user_email(self) -> str | None: ...
+
+
+class _UserRowsTable(Protocol):
+    async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_UserRow]: ...
+
+
 class _ShadowEvalJobRow(Protocol):
     @property
     def id(self) -> str: ...
 
+    @property
+    def group_id(self) -> str: ...
+
+    @property
+    def target_type(self) -> str: ...
+
+    @property
+    def target_id(self) -> str: ...
+
 
 class _ShadowEvalJobTable(Protocol):
     async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ShadowEvalJobRow]: ...
@@ -138,6 +172,14 @@ def _verification_tokens(prisma_client: "PrismaClient") -> _VerificationTokenTab
     return prisma_client.db.litellm_verificationtoken
 
 
+def _team_rows(prisma_client: "PrismaClient") -> _TeamRowsTable:
+    return prisma_client.db.litellm_teamtable
+
+
+def _user_rows(prisma_client: "PrismaClient") -> _UserRowsTable:
+    return prisma_client.db.litellm_usertable
+
+
 def _shadow_eval_jobs(prisma_client: "PrismaClient") -> _ShadowEvalJobTable:
     return prisma_client.db.litellm_shadowevaljob
 
@@ -836,7 +878,7 @@ def _validate_judge_is_not_a_candidate(
 
 
 def _is_unique_violation(error: Exception) -> bool:
-    """Whether a Prisma create failed on a unique index. One active job per key and
+    """Whether a Prisma create failed on a unique index. One active job per target and
     direction lives in a partial unique index (raw SQL in the migration; schema.prisma
     cannot express partial indexes), so the read-then-create check above it is advisory:
     two concurrent starts pass the read, and the loser must surface as the same 409
@@ -885,7 +927,7 @@ _ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT
 # direction, and mid-deploy rows from old pods price as judge-only until the deploy ends).
 _SWEEP_FINISHED_JOBS_SQL: Final = """
 UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = (NOW() AT TIME ZONE 'utc')
-WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL
+WHERE j.target_type = $2 AND j.target_id = ANY($1::text[]) AND j.stopped_at IS NULL
   AND (
     j.ends_at <= (NOW() AT TIME ZONE 'utc')
     OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns
@@ -966,10 +1008,10 @@ WHERE group_id IN (
 )
 """
 
-_LIST_LEGS_BY_KEY_SQL: Final = """
+_LIST_LEGS_BY_TARGET_SQL: Final = """
 SELECT * FROM "LiteLLM_ShadowEvalJob"
 WHERE group_id IN (
-    SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE api_key_id = $2
+    SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE target_type = $2 AND target_id = $3
     GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int
 )
 """
@@ -1007,15 +1049,16 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]:
 
 class _LegRow(BaseModel):
     """One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is
-    one key's leg of a job; the legs of a job share group_id and identical config, written
-    together by one create_many. The API's job id is the group id, so leg ids never leave
-    the server (attempts reference them internally)."""
+    one target's leg of a job; the legs of a job share group_id and identical config,
+    written together by one create_many. The API's job id is the group id, so leg ids
+    never leave the server (attempts reference them internally)."""
 
     model_config = ConfigDict(from_attributes=True)
 
     id: str
     group_id: str
-    api_key_id: str
+    target_type: ShadowEvalTargetType
+    target_id: str
     router_name: str
     direction: ShadowEvalDirection
     baseline_model: str | None = None
@@ -1068,16 +1111,17 @@ def _group_response(
     first: Final = legs[0]
     return ShadowEvalJobResponse(
         job_id=group_id,
-        keys=tuple(
-            ShadowEvalJobKeyResponse(
-                api_key_id=leg.api_key_id,
+        targets=tuple(
+            ShadowEvalJobTargetResponse(
+                target_type=leg.target_type,
+                target_id=leg.target_id,
                 max_turns=leg.max_turns,
                 max_budget=leg.max_budget,
                 stopped_at=leg.stopped_at,
                 attempt_count=stats.attempt_count if (stats := attempt_counts.get(leg.id)) else 0,
                 spend=round(stats.spend, 6) if stats else 0.0,
             )
-            for leg in sorted(legs, key=lambda leg: leg.api_key_id)
+            for leg in sorted(legs, key=lambda leg: (leg.target_type, leg.target_id))
         ),
         router_name=first.router_name,
         direction=first.direction,
@@ -1090,34 +1134,85 @@ def _group_response(
     )
 
 
-_NO_KEY_LABELS: Final[tuple[str | None, str | None]] = (None, None)
+_NO_TARGET_LABELS: Final[tuple[str | None, str | None]] = (None, None)
 
 
-async def _with_key_labels(
+def _target_labels(
+    key_rows: Sequence[_VerificationTokenRow],
+    team_rows: Sequence[_TeamRow],
+    user_rows: Sequence[_UserRow],
+) -> Mapping[tuple[str, str], tuple[str | None, str | None]]:
+    """Display labels by (target_type, target_id): a key's (alias, masked name), a
+    team's (alias, None), a user's (email, None)."""
+    return MappingProxyType(
+        {  # mutable-ok: MappingProxyType needs a dict to wrap
+            key: value
+            for key, value in chain(
+                ((("key", row.token), (row.key_alias, row.key_name)) for row in key_rows),
+                ((("team", row.team_id), (row.team_alias, None)) for row in team_rows),
+                ((("user", row.user_id), (row.user_email, None)) for row in user_rows),
+            )
+        }
+    )
+
+
+def _target_ids_of(responses: Sequence[ShadowEvalJobResponse], target_type: ShadowEvalTargetType) -> tuple[str, ...]:
+    return tuple(
+        sorted(
+            frozenset(
+                target.target_id
+                for response in responses
+                for target in response.targets
+                if target.target_type == target_type
+            )
+        )
+    )
+
+
+async def _with_target_labels(
     prisma_client: "PrismaClient", responses: Sequence[ShadowEvalJobResponse]
 ) -> tuple[ShadowEvalJobResponse, ...]:
-    """Resolve every scoped key's hash to its alias and masked name in one batched read,
-    so the UI can say whose traffic a job shadows. Deleted keys resolve to None."""
+    """Resolve every scoped target's id to a display label in one batched read per kind,
+    so the UI can say whose traffic a job shadows: a key's alias and masked name, a
+    team's alias, a user's email. Deleted targets resolve to None."""
     if not responses:
         return ()
-    tokens: Final = sorted(frozenset(key.api_key_id for response in responses for key in response.keys))
-    key_rows: Final = await _verification_tokens(prisma_client).find_many(
-        where={"token": {"in": tokens}}  # mutable-ok: Prisma filter
+    tokens: Final = _target_ids_of(responses, "key")
+    team_ids: Final = _target_ids_of(responses, "team")
+    user_ids: Final = _target_ids_of(responses, "user")
+    key_rows: Final = (
+        await _verification_tokens(prisma_client).find_many(
+            where={"token": {"in": list(tokens)}}  # mutable-ok: Prisma filter
+        )
+        if tokens
+        else ()
     )
-    labels: Final[Mapping[str, tuple[str | None, str | None]]] = {
-        row.token: (row.key_alias, row.key_name) for row in key_rows or ()
-    }
+    team_rows: Final = (
+        await _team_rows(prisma_client).find_many(
+            where={"team_id": {"in": list(team_ids)}}  # mutable-ok: Prisma filter
+        )
+        if team_ids
+        else ()
+    )
+    user_rows: Final = (
+        await _user_rows(prisma_client).find_many(
+            where={"user_id": {"in": list(user_ids)}}  # mutable-ok: Prisma filter
+        )
+        if user_ids
+        else ()
+    )
+    labels: Final = _target_labels(key_rows or (), team_rows or (), user_rows or ())
     return tuple(
         response.model_copy(
             update={  # mutable-ok: pydantic update payload
-                "keys": tuple(
-                    key.model_copy(
+                "targets": tuple(
+                    target.model_copy(
                         update={  # mutable-ok: pydantic update payload
-                            "key_alias": labels.get(key.api_key_id, _NO_KEY_LABELS)[0],
-                            "key_name": labels.get(key.api_key_id, _NO_KEY_LABELS)[1],
+                            "target_alias": labels.get((target.target_type, target.target_id), _NO_TARGET_LABELS)[0],
+                            "key_name": labels.get((target.target_type, target.target_id), _NO_TARGET_LABELS)[1],
                         }
                     )
-                    for key in response.keys
+                    for target in response.targets
                 )
             }
         )
@@ -1125,29 +1220,37 @@ async def _with_key_labels(
     )
 
 
-async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> ShadowEvalResult | None:
-    """All three stratifications of one job's verdicts. Tier answers "where does the router
-    do well"; the model stratification groups by whichever model served the real arm, so it
-    answers "which of the models these keys use today would the router beat" forward, and
-    "for the turns the router sent to X, did X beat the baseline" in reverse; key answers
-    "which key's traffic does the router suit". Reads are bounded by the job's own attempts
-    (<= the sum of its keys' max_turns) via the job_id index."""
+async def _shadow_eval_results(
+    prisma_client: "PrismaClient", legs: Sequence[_LegRow]
+) -> tuple[ShadowEvalResult | None, Mapping[tuple[str, str], ShadowEvalSlice]]:
+    """One job's stratified verdicts, plus each target's own slice keyed by the
+    (target_type, target_id) pair so a key, team, and user sharing an id can never
+    collapse into one entry. Tier answers "where does the router do well"; the model
+    stratification groups by whichever model served the real arm, so it answers "which
+    of the models these targets use today would the router beat" forward, and "for the
+    turns the router sent to X, did X beat the baseline" in reverse; the per-target
+    slices answer "which target's traffic does the router suit". Reads are bounded by
+    the job's own attempts (<= the sum of its targets' max_turns) via the job_id index."""
     leg_ids: Final = [leg.id for leg in legs]  # mutable-ok: query param
     by_tier: Final = _ATTEMPT_AGG_ROWS.validate_python(
         await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, leg_ids) or ()
     )
     if not by_tier:
-        return None
+        return None, MappingProxyType({})
     by_model: Final = _ATTEMPT_AGG_ROWS.validate_python(
         await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, leg_ids) or ()
     )
-    key_by_leg: Final = MappingProxyType({leg.id: leg.api_key_id for leg in legs})
+    target_by_leg: Final = MappingProxyType({leg.id: (leg.target_type, leg.target_id) for leg in legs})
     by_leg: Final = _ATTEMPT_AGG_ROWS.validate_python(
         await _query_raw(prisma_client, _ATTEMPT_AGG_BY_LEG_SQL, leg_ids) or ()
     )
-    by_key: Final = tuple(
-        row.model_copy(update={"grp": key_by_leg[row.grp]})  # mutable-ok: pydantic update payload
-        for row in by_leg
+    verdicts_by_target: Final[Mapping[tuple[str, str], ShadowEvalSlice]] = MappingProxyType(
+        {  # mutable-ok: MappingProxyType needs a dict to wrap
+            target_by_leg[slice.group]: slice.model_copy(
+                update={"group": target_by_leg[slice.group][1]}  # mutable-ok: pydantic update payload
+            )
+            for slice in _slices(by_leg)
+        }
     )
     total_turns: Final = sum(r.turn_count for r in by_tier)
     funnel_rows: Final = await _query_raw(prisma_client, _FUNNEL_TOTALS_SQL, leg_ids)
@@ -1155,10 +1258,9 @@ async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_Le
     # Coverage only when EVERY leg has a funnel row: a partial seed (one leg's insert
     # failed) must read as unknown, not as job-level counts missing a leg's traffic.
     funnel: Final = counted if counted is not None and counted.legs_with_rows == len(leg_ids) else None
-    return ShadowEvalResult(
+    result: Final = ShadowEvalResult(
         by_tier=_slices(by_tier),
         by_current_model=_slices(by_model),
-        by_key=_slices(by_key),
         overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns),
         overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns),
         sampled_real_spend=sum(r.real_spend for r in by_tier),
@@ -1168,6 +1270,7 @@ async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_Le
         shed_count=funnel.shed if funnel is not None else None,
         withheld_count=funnel.withheld if funnel is not None else None,
     )
+    return result, verdicts_by_target
 
 
 @router.post(
@@ -1182,22 +1285,29 @@ async def start_shadow_eval(
     user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
 ) -> ShadowEvalJobResponse:
     """
-    Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against
-    a second arm, judge the two responses blind, and stratify win rates by tier, by the model
-    that served the real arm, and by key.
+    Start a shadow eval: duplicate a sampled slice of one or more targets' live traffic
+    against a second arm, judge the two responses blind, and stratify win rates by tier,
+    by the model that served the real arm, and by target.
 
-    A forward job answers whether the keys should adopt router_name: it samples the requests
-    the router did not serve and duplicates them through it. A reverse job answers whether a
-    key already on the router still gains from it: it samples the requests the router did
-    serve and duplicates them against baseline_model. A key can hold one active job per
-    direction, so both questions can run at once.
+    A target is a virtual key, a team, or a user. Team and user targets match on the
+    identity every request resolves to at auth time, so they cover JWT-authenticated
+    traffic, which presents no virtual key; a user target samples that user's traffic
+    across all their teams, whether it arrives on a JWT or a key they own.
 
-    Shadow responses are never served to users. Each key samples until its recorded eval
-    spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's
-    window ends, or the job is stopped, so one key running out of budget does not end
-    sampling for the others; sampling changes propagate to pods within about 10 seconds.
-    Shadow and judge calls bill to the shadowed key but are excluded from request counts
-    and auto-router adoption metrics.
+    A forward job answers whether the targets should adopt router_name: it samples the
+    requests the router did not serve and duplicates them through it. A reverse job
+    answers whether a target already on the router still gains from it: it samples the
+    requests the router did serve and duplicates them against baseline_model. A target
+    can hold one active job per direction, so both questions can run at once, and a
+    request matching several jobs' targets (say its key and its team) is sampled by
+    each, separately budgeted.
+
+    Shadow responses are never served to users. Each target samples until its recorded
+    eval spend, the shadow and judge calls' own cost, reaches max_budget dollars, the
+    job's window ends, or the job is stopped, so one target running out of budget does
+    not end sampling for the others; sampling changes propagate to pods within about 10
+    seconds. Shadow and judge calls bill to the sampled request's own identity but are
+    excluded from request counts and auto-router adoption metrics.
     """
     from litellm.proxy.proxy_server import llm_router, prisma_client
 
@@ -1206,35 +1316,88 @@ async def start_shadow_eval(
         raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
     if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name):
         raise HTTPException(status_code=400, detail=f"'{data.router_name}' is not a configured auto-router")
-    token_rows: Final = await _verification_tokens(prisma_client).find_many(
-        where={"token": {"in": list(data.api_key_ids)}}  # mutable-ok: Prisma filter
-    )
-    unknown: Final = tuple(sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ())))
-    if unknown:
-        raise HTTPException(
-            status_code=400,
-            detail=(
-                f"api_key_ids not on this proxy: {', '.join(unknown)}; pass each key's token hash, "
-                "the value the key list and key info endpoints report"
-            ),
+    token_rows: Final = (
+        await _verification_tokens(prisma_client).find_many(
+            where={"token": {"in": list(data.api_key_ids)}}  # mutable-ok: Prisma filter
         )
+        if data.api_key_ids
+        else ()
+    )
+    team_rows: Final = (
+        await _team_rows(prisma_client).find_many(
+            where={"team_id": {"in": list(data.team_ids)}}  # mutable-ok: Prisma filter
+        )
+        if data.team_ids
+        else ()
+    )
+    user_rows: Final = (
+        await _user_rows(prisma_client).find_many(
+            where={"user_id": {"in": list(data.user_ids)}}  # mutable-ok: Prisma filter
+        )
+        if data.user_ids
+        else ()
+    )
+    unknown_keys: Final = sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ()))
+    unknown_teams: Final = sorted(frozenset(data.team_ids) - frozenset(row.team_id for row in team_rows or ()))
+    unknown_users: Final = sorted(frozenset(data.user_ids) - frozenset(row.user_id for row in user_rows or ()))
+    unknown_parts: Final = tuple(
+        part
+        for part in (
+            (
+                f"api_key_ids not on this proxy: {', '.join(unknown_keys)}; pass each key's token hash, "
+                "the value the key list and key info endpoints report"
+            )
+            if unknown_keys
+            else None,
+            f"team_ids not on this proxy: {', '.join(unknown_teams)}" if unknown_teams else None,
+            f"user_ids not on this proxy: {', '.join(unknown_users)}" if unknown_users else None,
+        )
+        if part is not None
+    )
+    if unknown_parts:
+        raise HTTPException(status_code=400, detail=". ".join(unknown_parts))
 
     # Every model check below runs once per team the job samples for, since that is the
     # identity the shadow and judge calls carry and therefore what the router selects on.
-    team_ids: Final = tuple(dict.fromkeys(row.team_id for row in token_rows or ()))
+    # A user target's traffic can span teams, so it validates unscoped (None); each
+    # sampled attempt still resolves the judge under its own request's team at eval time.
+    team_ids: Final = tuple(
+        dict.fromkeys(
+            (
+                *(row.team_id for row in token_rows or ()),
+                *data.team_ids,
+                *((None,) if data.user_ids else ()),
+            )
+        )
+    )
     _validate_plain_model(llm_router, data.judge_model, "judge_model", team_ids)
     if data.baseline_model is not None:
         _validate_plain_model(llm_router, data.baseline_model, "baseline_model", team_ids)
     _validate_judge_is_not_a_candidate(llm_router, data, team_ids)
 
+    requested_targets: Final[tuple[tuple[ShadowEvalTargetType, str], ...]] = (
+        *(("key", key) for key in data.api_key_ids),
+        *(("team", team) for team in data.team_ids),
+        *(("user", user) for user in data.user_ids),
+    )
+    requested_by_type: Final[tuple[tuple[ShadowEvalTargetType, tuple[str, ...]], ...]] = tuple(
+        (target_type, ids)
+        for target_type, ids in (("key", data.api_key_ids), ("team", data.team_ids), ("user", data.user_ids))
+        if ids
+    )
     # A job whose window passed or whose budget ran out stopped sampling on its own,
-    # but its legs still hold their slots in the per-key, per-direction partial unique index
-    # until stamped; free them so a new eval can start. Sweeping both directions is deliberate.
-    requested: Final = list(data.api_key_ids)  # mutable-ok: query param
-    await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, requested)
+    # but its legs still hold their slots in the per-target, per-direction partial unique
+    # index until stamped; free them so a new eval can start. Sweeping both directions is
+    # deliberate. Sweep and claim filter on exact (target_type, id) pairs so a team id
+    # that happens to equal a key hash never matches the other kind's slot.
+    for target_type, ids in requested_by_type:
+        await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, list(ids), target_type)  # mutable-ok: query param
     claimed: Final = await _shadow_eval_jobs(prisma_client).find_many(
         where={  # mutable-ok: Prisma filter
-            "api_key_id": {"in": requested},  # mutable-ok: Prisma filter
+            "OR": [  # mutable-ok: Prisma filter
+                {"target_type": target_type, "target_id": {"in": list(ids)}}  # mutable-ok: Prisma filter
+                for target_type, ids in requested_by_type
+            ],
             "direction": data.direction,
             "stopped_at": None,
         },
@@ -1244,7 +1407,7 @@ async def start_shadow_eval(
             status_code=409,
             detail=(
                 f"Already in an active {data.direction} shadow eval job: "
-                + ", ".join(sorted(f"{row.api_key_id} (job {row.group_id})" for row in claimed))
+                + ", ".join(sorted(f"{row.target_type} {row.target_id} (job {row.group_id})" for row in claimed))
                 + ". Stop it first."
             ),
         )
@@ -1268,10 +1431,16 @@ async def start_shadow_eval(
         # Leg ids are minted here rather than by the DB default so the funnel seed below
         # writes from the same values with no read-back, which a lagging read replica
         # (DATABASE_URL_READ_REPLICA) could otherwise return empty.
-        leg_ids: Final = tuple(str(uuid4()) for _ in data.api_key_ids)
+        leg_ids: Final = tuple(str(uuid4()) for _ in requested_targets)
         await _shadow_eval_jobs(prisma_client).create_many(
             data=[  # mutable-ok: Prisma payload
-                {**shared_config, "id": leg_id, "api_key_id": key} for leg_id, key in zip(leg_ids, data.api_key_ids)
+                {  # mutable-ok: Prisma payload
+                    **shared_config,
+                    "id": leg_id,
+                    "target_type": target_type,
+                    "target_id": target_id,
+                }  # mutable-ok: Prisma payload
+                for leg_id, (target_type, target_id) in zip(leg_ids, requested_targets)
             ]
         )
     except Exception as e:
@@ -1280,7 +1449,8 @@ async def start_shadow_eval(
         raise HTTPException(
             status_code=409,
             detail=(
-                f"A requested key was claimed by another {data.direction} shadow eval job concurrently. Stop it first."
+                f"A requested target was claimed by another {data.direction} shadow eval job concurrently. "
+                "Stop it first."
             ),
         ) from e
     # Seed a zero funnel row per leg NOW: a fully covered job never skips a request, so
@@ -1293,18 +1463,19 @@ async def start_shadow_eval(
         )
     except Exception as seed_err:  # noqa: BLE001  # coverage is advisory; the job must still start
         verbose_proxy_logger.error("shadow_eval: funnel seed failed for job %s: %s", group_id, seed_err)
-    labels: Final = MappingProxyType({row.token: row for row in token_rows})
+    labels: Final = _target_labels(token_rows or (), team_rows or (), user_rows or ())
     return ShadowEvalJobResponse(
         job_id=group_id,
-        keys=tuple(
-            ShadowEvalJobKeyResponse(
-                api_key_id=api_key_id,
+        targets=tuple(
+            ShadowEvalJobTargetResponse(
+                target_type=target_type,
+                target_id=target_id,
                 max_turns=SHADOW_EVAL_TURN_VALVE,
                 max_budget=data.max_budget,
-                key_alias=labels[api_key_id].key_alias,
-                key_name=labels[api_key_id].key_name,
+                target_alias=labels.get((target_type, target_id), _NO_TARGET_LABELS)[0],
+                key_name=labels.get((target_type, target_id), _NO_TARGET_LABELS)[1],
             )
-            for api_key_id in sorted(data.api_key_ids)
+            for target_type, target_id in sorted(requested_targets)
         ),
         router_name=data.router_name,
         direction=data.direction,
@@ -1324,22 +1495,29 @@ async def start_shadow_eval(
 )
 async def list_shadow_eval_jobs(
     user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
-    api_key_id: Annotated[
-        str | None, Query(description="Filter to jobs that shadow this key, alone or alongside others")
+    target_type: Annotated[
+        ShadowEvalTargetType | None, Query(description="Kind of target to filter on; requires target_id")
+    ] = None,
+    target_id: Annotated[
+        str | None, Query(description="Filter to jobs that shadow this target, alone or alongside others")
     ] = None,
     limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50,
 ) -> tuple[ShadowEvalJobResponse, ...]:
-    """List shadow eval jobs, newest first, each key with its attempt count so status is
-    accurate. Judged counts, spend, and results ride the detail endpoint only."""
+    """List shadow eval jobs, newest first, each target with its attempt count so status
+    is accurate. Judged counts, spend, and results ride the detail endpoint only."""
     from litellm.proxy.proxy_server import prisma_client
 
     _require_admin_viewer(user_api_key_dict, "view shadow evals")
     if prisma_client is None:
         raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
+    filter_type: Final = target_type if isinstance(target_type, str) else None
+    filter_id: Final = target_id if isinstance(target_id, str) else None
+    if (filter_type is None) != (filter_id is None):
+        raise HTTPException(status_code=400, detail="target_type and target_id filter together; pass both or neither")
     legs: Final = _LEG_ROWS.validate_python(
         (
-            await _query_raw(prisma_client, _LIST_LEGS_BY_KEY_SQL, limit, api_key_id)
-            if api_key_id
+            await _query_raw(prisma_client, _LIST_LEGS_BY_TARGET_SQL, limit, filter_type, filter_id)
+            if filter_type and filter_id
             else await _query_raw(prisma_client, _LIST_LEGS_SQL, limit)
         )
         or ()
@@ -1354,7 +1532,7 @@ async def list_shadow_eval_jobs(
         by_group, key=lambda group_id: max(leg.created_at for leg in by_group[group_id]), reverse=True
     )
     counts: Final = await _leg_attempt_counts(prisma_client, legs)
-    return await _with_key_labels(
+    return await _with_target_labels(
         prisma_client, tuple(_group_response(group_id, by_group[group_id], counts) for group_id in newest_first)
     )
 
@@ -1391,16 +1569,25 @@ async def get_shadow_eval_job(
         where={"job_id": {"in": leg_ids}, "outcome": "error"},  # mutable-ok: Prisma filter
         order={"created_at": "desc"},  # mutable-ok: Prisma order
     )
-    labeled: Final = await _with_key_labels(
+    labeled: Final = await _with_target_labels(
         prisma_client, (_group_response(job_id, legs, await _leg_attempt_counts(prisma_client, legs)),)
     )
+    results, verdicts_by_target = await _shadow_eval_results(prisma_client, legs)
     return labeled[0].model_copy(
         update={  # mutable-ok: pydantic update payload
             "judged_count": totals[0].judged_count if totals else 0,
             "error_count": totals[0].error_count if totals else 0,
             "judge_spend": round(totals[0].judge_spend, 6) if totals else 0.0,
             "last_error": latest_error.error if latest_error else None,
-            "results": await _shadow_eval_results(prisma_client, legs),
+            "results": results,
+            "targets": tuple(
+                target.model_copy(
+                    update={  # mutable-ok: pydantic update payload
+                        "verdicts": verdicts_by_target.get((target.target_type, target.target_id))
+                    }
+                )
+                for target in labeled[0].targets
+            ),
         }
     )
 
@@ -1415,8 +1602,8 @@ async def stop_shadow_eval_job(
     job_id: str,
     user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
 ) -> ShadowEvalJobResponse:
-    """Stop an active shadow eval job, every key it scopes at once. Attempts are kept;
-    sampling halts within ~10s. Keys that already stopped on their own budget keep the
+    """Stop an active shadow eval job, every target it scopes at once. Attempts are kept;
+    sampling halts within ~10s. Targets that already stopped on their own budget keep the
     stopped_at they earned. The statement is the whole state machine: it claims the job
     only while a leg still samples inside the window with no stop recorded, so a racing
     operator, a same-instant budget spend, and a repeat stop all read the same 400 with
@@ -1443,5 +1630,5 @@ async def stop_shadow_eval_job(
     current: Final = _group_response(job_id, legs, counts)
     if claimed == 0:
         raise HTTPException(status_code=400, detail=f"Job {job_id} is already {current.status}")
-    labeled: Final = await _with_key_labels(prisma_client, (current,))
+    labeled: Final = await _with_target_labels(prisma_client, (current,))
     return labeled[0]

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -19,13 +19,13 @@ import re
 import secrets
 import traceback
 from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
-from contextlib import AbstractAsyncContextManager
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypeVar, cast
 
 import fastapi
 import yaml
 from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -155,6 +155,7 @@ from litellm.types.utils import (
 )
 
 if TYPE_CHECKING:
+    import prisma
     from prisma import Prisma
     from prisma import models as prisma_models
 
@@ -182,6 +183,14 @@ class _TxTables(Protocol):
     litellm_proxymodeltable: TableActions[object]
 
 
+class _ModelParamsUpdate(TypedDict):
+    litellm_params: ReadOnly["prisma.Json"]
+
+
+class _ModelRowWhere(TypedDict):
+    model_id: ReadOnly[str]
+
+
 class _ConfigTableActions(Protocol):
     """Config table surface this module needs; the shared repository seam exposes no ``update``."""
 
@@ -273,12 +282,6 @@ def _env_vars_param_value(param: _EnvVarsParam) -> Mapping[str, str] | None:
     return param.param_value
 
 
-def _tx_tables_context(
-    open_tx: Callable[[], AbstractAsyncContextManager[_TxTables]],
-) -> AbstractAsyncContextManager[_TxTables]:
-    return open_tx()
-
-
 async def _check_custom_key_allowed(custom_key_value: str | None) -> None:
     """Raise 403 if custom API keys are disabled and a custom key was provided."""
     if custom_key_value is None:
@@ -4484,27 +4487,29 @@ async def _rotate_master_key(
     if models:
         decrypted_models: Final = proxy_config.decrypt_model_list_from_db(new_models=models)
         verbose_proxy_logger.debug("ABLE TO DECRYPT MODELS - len(decrypted_models): %s", len(decrypted_models))
-        new_models: Final[list[dict[str, object]]] = []
-        for model in decrypted_models:
-            new_model = await _add_model_to_db(
-                model_params=Deployment(**model),
-                user_api_key_dict=user_api_key_dict,
-                prisma_client=prisma_client,
-                new_encryption_key=new_master_key,
-                should_create_model_in_db=False,
-            )
-            if new_model:
-                _dumped = dict[str, object](_as_object_dict(new_model.model_dump(exclude_none=True)))
-                _dumped["litellm_params"] = prisma.Json(_dumped["litellm_params"])
-                _dumped["model_info"] = prisma.Json(_dumped["model_info"])
-                new_models.append(_dumped)
-        verbose_proxy_logger.debug("Resetting proxy model table")
-        async with _tx_tables_context(prisma_client.db.tx) as tx:
-            await tx.litellm_proxymodeltable.delete_many()
-            verbose_proxy_logger.debug("Creating %s models", len(new_models))
-            await tx.litellm_proxymodeltable.create_many(
-                data=new_models,
-            )
+        reencrypted_models: Final = tuple(
+            [
+                reencrypted
+                for model in decrypted_models
+                if (
+                    reencrypted := await _add_model_to_db(
+                        model_params=Deployment(**model),
+                        user_api_key_dict=user_api_key_dict,
+                        prisma_client=prisma_client,
+                        new_encryption_key=new_master_key,
+                        should_create_model_in_db=False,
+                    )
+                )
+            ]
+        )
+        verbose_proxy_logger.debug("Re-encrypting litellm_params on %s model rows", len(reencrypted_models))
+        async with prisma_client.db.tx(timeout=timedelta(minutes=2)) as tx_ctx:
+            tx: Final[_TxTables] = tx_ctx
+            for reencrypted_model in reencrypted_models:
+                await tx.litellm_proxymodeltable.update_many(
+                    data=_ModelParamsUpdate(litellm_params=prisma.Json(reencrypted_model.litellm_params)),
+                    where=_ModelRowWhere(model_id=reencrypted_model.model_id),
+                )
         await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable")
     # 3. process config table
     try:

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -6,6 +6,8 @@ Provider-specific Pass-Through Endpoints
 Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc.
 """
 
+from __future__ import annotations
+
 import hmac
 import json
 import os
@@ -28,6 +30,7 @@ from litellm.constants import (
 )
 from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.llms.anthropic.common_utils import AnthropicModelInfo
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.proxy._types import *
 from litellm.proxy.auth.handle_jwt import JWTHandler
@@ -51,6 +54,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     create_websocket_passthrough_route,
     websocket_passthrough_request,
 )
+from litellm.proxy.utils import ProxyLogging as ProxyLoggingType
 from litellm.proxy.utils import is_known_model
 from litellm.proxy.vector_store_endpoints.utils import (
     assert_proxy_admin_for_vector_store_index_management,
@@ -70,13 +74,17 @@ from litellm.utils import ProviderConfigManager
 from .passthrough_endpoint_router import PassthroughEndpointRouter
 
 if TYPE_CHECKING:
+    from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig
     from litellm.router import Router
 
+    ProxyConfig = _ProxyConfig  # rebind-ok: conditional type alias
+else:
+    ProxyConfig = Any  # rebind-ok: runtime fallback
+
 vertex_llm_base: Final = VertexBase()
 router: Final = APIRouter()
 openai_passthrough_router: Final = APIRouter()
 default_vertex_config: Final = None
-
 passthrough_endpoint_router: Final = PassthroughEndpointRouter()
 
 
@@ -495,8 +503,14 @@ async def milvus_proxy_route(
     request_body: Final = await get_request_body(request)
 
     # check collectionName
-    collection_name: Final = cast(str | None, request_body.get("collectionName"))
-    extra_headers = {}
+    _raw_collection_name: Final = request_body.get("collectionName")
+    if _raw_collection_name is not None and not isinstance(_raw_collection_name, str):
+        raise HTTPException(
+            status_code=400,
+            detail=f"collectionName must be a string. Got {type(_raw_collection_name).__name__}",
+        )
+    collection_name: str | None = _raw_collection_name  # rebind-ok: locally scoped conversion
+    extra_headers = {}  # mutable-ok: dict for extra headers; rebind-ok: reassigned later from credentials
     base_target_url: str | None = None
     if not collection_name:
         raise HTTPException(
@@ -1273,7 +1287,7 @@ def _resolve_vertex_model_from_router(
         vertex_location: Current vertex location (may be from URL)
 
     Returns:
-        Tuple of (encoded_endpoint, endpoint, vertex_project, vertex_location)
+        tuple of (encoded_endpoint, endpoint, vertex_project, vertex_location)
         with resolved values from router config
     """
     if not llm_router:
@@ -1702,7 +1716,7 @@ def get_vertex_ai_allowed_incoming_headers(request: Request) -> dict:
 
 
 def get_vertex_pass_through_handler(
-    call_type: Literal["discovery", "aiplatform"],
+    call_type: Literal["discovery", "aiplatform"],  # noqa: UP037
 ) -> BaseVertexAIPassThroughHandler:
     if call_type == "discovery":
         return VertexAIDiscoveryPassThroughHandler()
@@ -1726,7 +1740,7 @@ def _override_vertex_params_from_router_credentials(
         vertex_location: Current vertex location (from URL)
 
     Returns:
-        Tuple of (vertex_project, vertex_location) with overridden values if applicable
+        tuple of (vertex_project, vertex_location) with overridden values if applicable
     """
     if router_credentials is None:
         return vertex_project, vertex_location
@@ -1893,12 +1907,12 @@ async def _prepare_vertex_auth_headers(
             authenticated them is stripped on the credential-less branch
 
     Returns:
-        Tuple containing:
+        tuple containing:
             - headers: dict - Authentication headers to use
-            - base_target_url: Optional[str] - Updated base target URL
+            - base_target_url: str | None - Updated base target URL
             - headers_passed_through: bool - Whether headers were passed through from request
-            - vertex_project: Optional[str] - Updated vertex project ID
-            - vertex_location: Optional[str] - Updated vertex location
+            - vertex_project: str | None - Updated vertex project ID
+            - vertex_location: str | None - Updated vertex location
     """
     vertex_llm_base: Final = VertexBase()
     headers_passed_through = False
@@ -2546,7 +2560,7 @@ def _vertex_publisher_model_suffix(model: str) -> str:
     return f"{VERTEX_PUBLISHER_MODEL_PREFIX}{model.rsplit('/', 1)[-1]}"
 
 
-def _get_llm_router() -> "Router | None":
+def _get_llm_router() -> Router | None:
     from litellm.proxy.proxy_server import llm_router
 
     return llm_router
@@ -2586,7 +2600,7 @@ def _resolve_vertex_live_credentials(
 def _build_vertex_live_setup_model_rewriter(
     vertex_project: str | None,
     vertex_location: str | None,
-    llm_router: "Router | None",
+    llm_router: Router | None,
 ) -> Callable[[str], str] | None:
     """
     Rewrite the ``setup`` frame's model into the full Vertex resource path the Live API requires.
@@ -2606,7 +2620,7 @@ def _build_vertex_live_setup_model_rewriter(
     return rewrite
 
 
-def _resolve_alias_to_upstream_model(setup_model: str, llm_router: "Router | None") -> str:
+def _resolve_alias_to_upstream_model(setup_model: str, llm_router: Router | None) -> str:
     """
     The Live SDK wraps whatever the caller typed as ``models/<name>``, so a gateway alias arrives prefixed
     """
@@ -2796,6 +2810,238 @@ def create_generic_websocket_passthrough_endpoint(
     )
 
 
+@router.api_route(
+    "/gigachat/{endpoint:path}",
+    methods=["GET", "POST", "PUT", "DELETE", "PATCH"],  # mutable-ok: FastAPI route methods
+    tags=["Gigachat Pass-through", "pass-through"],  # mutable-ok: FastAPI route tags
+)
+async def gigachat_proxy_route(
+    endpoint: str,
+    request: Request,
+    fastapi_response: Response,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+) -> Response:
+    """
+    [Docs](https://docs.litellm.ai/docs/pass_through/gigachat)
+    """
+    from litellm.proxy.proxy_server import (
+        general_settings,
+        llm_router,
+        proxy_config,
+        proxy_logging_obj,
+        select_data_generator,
+        user_api_base,
+        user_max_tokens,
+        user_model,
+        user_request_timeout,
+        user_temperature,
+        version,
+    )
+
+    ## check for streaming
+    request_body: Final[dict[str, object]] = await get_request_body(request)
+    is_router_model = False  # rebind-ok: conditionally set to True when model uses router
+
+    raw_model: Final = request_body.get("model")
+    model: Final = raw_model if isinstance(raw_model, str) else None
+    if model:
+        is_router_model = is_passthrough_request_using_router_model(
+            request_body, llm_router
+        )  # rebind-ok: conditionally set to True
+    elif any(word in endpoint for word in ("completions", "embeddings")):
+        raise HTTPException(
+            status_code=400, detail={"error": "Model is required in request body"}
+        )  # mutable-ok: HTTPException detail dict
+
+    # If router model, use dedicated router passthrough handler
+    # This uses the same common processing path as non-router models
+    if model and is_router_model and llm_router:
+        return await handle_gigachat_passthrough_router_model(
+            model=model,
+            endpoint=endpoint,
+            request=request,
+            request_body=request_body,
+            fastapi_response=fastapi_response,
+            llm_router=llm_router,
+            user_api_key_dict=user_api_key_dict,
+            proxy_logging_obj=proxy_logging_obj,
+            general_settings=general_settings,
+            proxy_config=proxy_config,
+            select_data_generator=select_data_generator,
+            user_model=user_model,
+            user_temperature=user_temperature,
+            user_request_timeout=user_request_timeout,
+            user_max_tokens=user_max_tokens,
+            user_api_base=user_api_base,
+            version=version,
+        )
+
+    verbose_proxy_logger.debug(
+        "Gigachat passthrough: Using direct Gigachat model '%s' for endpoint '%s'", model, endpoint
+    )
+
+    from litellm.llms.gigachat.authenticator import get_access_token
+    from litellm.llms.gigachat.utils import GIGACHAT_BASE_URL
+
+    base_target_url: Final = get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL
+    request_path: Final = httpx.URL(endpoint).path
+    encoded_endpoint: Final = request_path if request_path.startswith("/") else f"/{request_path}"
+
+    base_url: Final = httpx.URL(base_target_url)
+    updated_url: Final = base_url.copy_with(
+        path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
+    )
+
+    is_streaming_request: Final = await is_streaming_request_fn(request)
+
+    endpoint_func: Final = create_pass_through_route(
+        endpoint=endpoint,
+        target=str(updated_url),
+        custom_headers={"Authorization": f"Bearer {get_access_token()}"},
+        is_streaming_request=is_streaming_request,
+    )
+    return await endpoint_func(
+        request,
+        fastapi_response,
+        user_api_key_dict,
+    )
+
+
+async def handle_gigachat_passthrough_router_model(
+    model: str,
+    endpoint: str,
+    request: Request,
+    request_body: dict,
+    fastapi_response: Response,
+    llm_router: litellm.Router,
+    user_api_key_dict: UserAPIKeyAuth,
+    proxy_logging_obj: ProxyLoggingType,
+    general_settings: dict,
+    proxy_config: ProxyConfig,
+    select_data_generator: Callable,
+    user_model: str | None,
+    user_temperature: float | None,
+    user_request_timeout: float | None,
+    user_max_tokens: int | None,
+    user_api_base: str | None,
+    version: str | None,
+) -> Response | StreamingResponse:
+    """
+    Handle Gigachat passthrough for router models (models defined in config.yaml).
+
+    Uses the same common processing path as non-router models to ensure
+    metadata and hooks are properly initialized.
+
+    Args:
+        model: The router model name (e.g., "gigachat/gigachat-2")
+        endpoint: The Gigachat endpoint path (e.g., "/chat/completions")
+        request: The FastAPI request object
+        request_body: The parsed request body
+        llm_router: The LiteLLM router instance
+        user_api_key_dict: The user API key authentication dictionary
+        proxy_logging_obj: Proxy logging
+        general_settings: Proxy general settings
+        proxy_config: Proxy config
+        select_data_generator: Select data generator function
+        (additional args for common processing)
+
+    Returns:
+        Response or StreamingResponse depending on endpoint type
+    """
+    from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
+
+    # Detect streaming based on request body
+    is_streaming: Final = request_body.get("stream", False)  # pyright: ignore[reportUnknownVariableType]  # request_body is dict[Unknown, Unknown]
+
+    data: dict[str, Any] = await _read_request_body(
+        request=request
+    )  # mutable-ok: mutated in place by proxy pipeline; pyright: ignore[reportExplicitAny]  # Any needed for proxy pipeline
+    if user_api_key_dict is not None:
+        auth_metadata: Final = {
+            metadata_key: value
+            for metadata_key, value in (
+                ("user_api_key_user_id", getattr(user_api_key_dict, "user_id", None)),
+                ("user_api_key_team_id", getattr(user_api_key_dict, "team_id", None)),
+                ("user_api_key_org_id", getattr(user_api_key_dict, "org_id", None)),
+                ("agent_id", getattr(user_api_key_dict, "agent_id", None)),
+            )
+            if value is not None
+        }
+        existing_metadata: Final = data.get("metadata")
+        data["metadata"] = {
+            **(existing_metadata if isinstance(existing_metadata, dict) else {}),
+            **auth_metadata,
+        }
+
+    verbose_proxy_logger.debug(
+        "Gigachat router passthrough: model='%s', endpoint='%s', streaming=%s", model, endpoint, is_streaming
+    )
+
+    # Use the common processing path (same as non-router models)
+    # This ensures all metadata, hooks, and logging are properly initialized
+
+    data["model"] = model
+    data["method"] = request.method
+    data["endpoint"] = endpoint
+    data["json"] = request_body
+    data["custom_llm_provider"] = "gigachat"
+
+    # Remove sensitive keys from data
+    keys: Final = [  # mutable-ok: list of keys to remove from data
+        "gigachat_auth_url",
+        "gigachat_access_token",
+        "gigachat_scope",
+        "api_base",
+        "api_key",
+    ]
+    for key in keys:
+        data.pop(key, None)
+
+    client: Final = get_async_httpx_client(
+        llm_provider=LlmProviders.GIGACHAT,
+        params={  # mutable-ok: httpx client params
+            "timeout": httpx.Timeout(timeout=600.0, connect=5.0),
+        },
+    )
+
+    data["client"] = client
+    base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
+
+    # Use the common passthrough processing to handle metadata and hooks
+    # This also handles all response formatting (streaming/non-streaming) and exceptions
+    try:
+        result = await base_llm_response_processor.base_passthrough_process_llm_request(  # rebind-ok: assigned once in try block
+            request=request,
+            fastapi_response=fastapi_response,
+            user_api_key_dict=user_api_key_dict,
+            proxy_logging_obj=proxy_logging_obj,
+            llm_router=llm_router,
+            general_settings=general_settings,
+            proxy_config=proxy_config,
+            select_data_generator=select_data_generator,
+            model=model,
+            user_model=user_model,
+            user_temperature=user_temperature,
+            user_request_timeout=user_request_timeout,
+            user_max_tokens=user_max_tokens,
+            user_api_base=user_api_base,
+            version=version,
+        )
+    except Exception as e:  # noqa: BLE001 # Safe catch-all for handle exception
+        # Use common exception handling
+        raise await base_llm_response_processor._handle_llm_api_exception(
+            e=e,
+            user_api_key_dict=user_api_key_dict,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+    else:
+        if isinstance(result, StreamingResponse):
+            if result.headers.get("Content-Type") is None:
+                result.headers["Content-Type"] = "text/event-stream; charset=utf-8"
+
+        return result
+
+
 @router.api_route(
     "/watsonx/{endpoint:path}",
     methods=["GET", "POST", "PUT", "DELETE", "PATCH"],

litellm/proxy/proxy_server.py

@@ -2658,7 +2658,6 @@ async def increment_spend_counters(
     budget_reservation: dict | None = None,
     end_user_id: str | None = None,
     tags: list[str] | None = None,
-    request_id: str | None = None,
     request_started_at: datetime | None = None,
     model_access_groups: Sequence[str] | None = None,
 ):
@@ -2733,7 +2732,6 @@ async def increment_spend_counters(
                 window_duration=duration,
                 window_start=key_window_start,
                 increment=cost,
-                request_id=request_id,
                 request_started_at=request_started_at,
             )
 
@@ -2777,7 +2775,6 @@ async def increment_spend_counters(
                 window_duration=duration,
                 window_start=team_window_start,
                 increment=cost,
-                request_id=request_id,
                 request_started_at=request_started_at,
             )
 
@@ -3005,16 +3002,15 @@ async def _enqueue_window_spend_row_update(
     window_duration: str,
     window_start: datetime | None,
     increment: float,
-    request_id: str | None,
     request_started_at: datetime | None,
 ) -> None:
     """Queue this request's cost against the LiteLLM_BudgetWindowSpend row for
     the window, so enforcement can read a maintained total instead of
     aggregating LiteLLM_SpendLogs.
 
-    request_id is the LiteLLM_SpendLogs id this cost was recorded under and
-    request_started_at its startTime; the flush uses them to keep the one-time
-    seed from counting a request that its increment already covers.
+    request_started_at is this request's LiteLLM_SpendLogs startTime; the flush
+    stops the one-time seed there so a request its increment already covers is
+    not counted twice.
 
     Enqueued even when the cache increment was skipped for a reserved counter:
     the reservation only pre-charged the counter, and the row still owes the
@@ -3035,7 +3031,6 @@ async def _enqueue_window_spend_row_update(
                 window_duration=window_duration,
                 window_start=window_start,
                 spend=increment,
-                request_id=request_id,
                 started_at=request_started_at,
             )
         )

litellm/proxy/public_endpoints/provider_create_fields.json

@@ -1318,6 +1318,68 @@
     ],
     "default_model_placeholder": "gpt-3.5-turbo"
   },
+  {
+    "provider": "GIGACHAT",
+    "provider_display_name": "GigaChat",
+    "litellm_provider": "gigachat",
+    "credential_fields": [
+      {
+        "key": "api_base",
+        "label": "API Base",
+        "placeholder": null,
+        "tooltip": null,
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "api_key",
+        "label": "API Key",
+        "placeholder": null,
+        "tooltip": null,
+        "required": false,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "gigachat_scope",
+        "label": "Scope",
+        "placeholder": null,
+        "tooltip": null,
+        "required": false,
+        "field_type": "select",
+        "options": [
+          "GIGACHAT_API_PERS",
+          "GIGACHAT_API_B2B",
+          "GIGACHAT_API_CORP"
+        ],
+        "default_value": "GIGACHAT_API_PERS"
+      },
+      {
+        "key": "gigachat_auth_url",
+        "label": "Auth URL",
+        "placeholder": null,
+        "tooltip": null,
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "gigachat_access_token",
+        "label": "Access token",
+        "placeholder": null,
+        "tooltip": "Disable OAuth, provide value to authorization.",
+        "required": false,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      }
+    ],
+    "default_model_placeholder": "GigaChat-2"
+  },
   {
     "provider": "GITHUB",
     "provider_display_name": "Github",

litellm/proxy/response_polling/background_streaming.py

@@ -144,10 +144,8 @@ async def background_streaming_task(
 
         # Process streaming response following OpenAI events format
         # https://platform.openai.com/docs/api-reference/responses-streaming
-        output_items: Final = dict[str, _OutputItem]()  # Track output items by ID
-        accumulated_text: Final = dict[
-            tuple[str, int], str
-        ]()  # Track accumulated text deltas by (item_id, content_index)
+        output_items: Final = dict[str, _OutputItem]()
+        accumulated_text: Final = dict[tuple[str, int], str]()
 
         # ResponsesAPIResponse fields to extract from response.completed
         usage_data = None
@@ -262,7 +260,6 @@ async def background_streaming_task(
                                 if "content" in delta_item:
                                     content_list = delta_item["content"]
                                     if content_index < len(content_list):
-                                        # Update existing content part with accumulated text
                                         content_entry = content_list[content_index]
                                         if isinstance(content_entry, dict):
                                             content_entry["text"] = accumulated_text[key]

litellm/proxy/schema.prisma

@@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession {
 model LiteLLM_ShadowEvalJob {
   id                String   @id @default(cuid())
   group_id          String   // legs of one job share this; the API's job id
-  api_key_id        String   // hashed virtual key whose traffic this leg shadows
+  target_type       String   @default("key") // key | team | user
+  target_id         String   // hashed virtual key, team_id, or user_id whose traffic this leg shadows
   router_name       String   // the auto-router under evaluation, in either direction
   direction         String   @default("forward") // forward | reverse
   baseline_model    String?  // reverse only: the fixed model the router is judged against
   judge_model       String
   shadow_percentage Float
   max_turns         Int      // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
-  max_budget        Float?   // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
+  max_budget        Float?   // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
   created_at        DateTime @default(now())
   created_by        String?
   ends_at           DateTime
@@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob {
   stopped_by        String?  // operator who stopped it early; null when it ended on its own
 
   @@index([group_id])
-  @@index([api_key_id])
+  @@index([target_type, target_id])
   @@index([created_at])
 }
 

litellm/rag/ingestion/vertex_ai_ingestion.py

@@ -186,7 +186,6 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase):
         base_url: Final = get_vertex_base_url(self.location)
         url: Final = f"{base_url}/v1beta1/projects/{self.project_id}/locations/{self.location}/ragCorpora"
 
-        # Build request body with camelCase keys (Vertex AI API format)
         vector_db_config: Final = self.vector_store_config.get("vector_db_config")
         embedding_model: Final = self.vector_store_config.get("embedding_model")
         embedding_model_config: Final = (
@@ -447,7 +446,6 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase):
         # Add max embedding requests per minute if specified
         max_embedding_qpm: Final = self.vector_store_config.get("max_embedding_requests_per_min")
 
-        # Build request body with camelCase keys (Vertex AI API format)
         chunking_config: Final = (
             {"chunkSize": chunk_size or 1024, "chunkOverlap": chunk_overlap or 200}
             if chunk_size or chunk_overlap

litellm/router.py

@@ -6486,6 +6486,8 @@ class Router:
                     **kwargs,
                 )
             elif call_type == "allm_passthrough_route":
+                if client:
+                    kwargs["client"] = client
                 return await self._ageneric_api_call_with_fallbacks(
                     original_function=original_function,
                     passthrough_on_no_deployment=True,
@@ -9152,7 +9154,8 @@ class Router:
             if _deployment_on_router is not None:
                 # deployment with this model_id exists on the router
                 if (
-                    deployment.litellm_params == _deployment_on_router.litellm_params
+                    deployment.model_name == _deployment_on_router.model_name
+                    and deployment.litellm_params == _deployment_on_router.litellm_params
                     and deployment.model_info == _deployment_on_router.model_info
                 ):
                     # No need to update

litellm/router_strategy/complexity_router/complexity_router.py

@@ -479,6 +479,33 @@ def _last_human_ask_index(
     )
 
 
+def _newest_turn_is_human_ask(
+    messages: Sequence[Mapping[str, object]] | None,
+    marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
+) -> bool:
+    """Whether the request's newest turn carries a real human ask, i.e. this is a new ask rather
+    than an agent loop's continuation traffic.
+
+    Anchored on `_last_human_ask_index` so every surface's plumbing reads as a continuation:
+    chat-completions tool turns are role=tool, Messages-surface tool_result turns flatten to empty
+    human text, and a hybrid turn carrying an ask alongside a tool_result still counts as an ask.
+    Compared against the newest non-system message rather than the raw tail, because Claude Code
+    appends a system-role reminder after the human turn; that trailing plumbing is neither an ask
+    nor loop traffic and must not turn a fresh ask into a continuation. An unreadable request (no
+    messages) is treated as a continuation: there is no ask to classify, which is the same reading
+    `_extract_current_ask_and_system_prompt` gives it downstream.
+    """
+    if not messages:
+        return False
+    newest_non_system: Final = next(
+        (index for index in range(len(messages) - 1, -1, -1) if messages[index].get("role") != "system"),
+        None,
+    )
+    if newest_non_system is None:
+        return False
+    return _last_human_ask_index(messages, marker_pairs) == newest_non_system
+
+
 def _iter_system_scope_texts(
     body_system: object,
     messages: Sequence[Mapping[str, object]],
@@ -706,11 +733,20 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo
     of the three: an agent names the conversation on its first turn, so the cheapest tier would be
     the pin every session starts with, and the real work that follows would run there for the whole
     TTL. It describes what that one call is, never what the session's traffic looks like.
+
+    A context-window escalation describes the prompt's size, not the session's complexity, and
+    size shrinks again the moment the client compacts: pinning the escalated tier would hold the
+    session on the big-window model long after the oversized context that forced it is gone. The
+    gate re-fires per request, so leaving these unpinned costs nothing but the classifier call.
     """
-    return decision is None or decision.get("cause") not in (
-        "default_model_fallback",
-        "plan_mode",
-        "housekeeping",
+    return decision is None or (
+        decision.get("cause")
+        not in (
+            "default_model_fallback",
+            "plan_mode",
+            "housekeeping",
+        )
+        and not decision.get("context_escalated")
     )
 
 
@@ -759,6 +795,39 @@ class ClassificationOutcome(NamedTuple):
     classifier_cost: float | None = None
 
 
+def _allowed(models: tuple[str, ...], fit_filter: frozenset[str] | None) -> tuple[str, ...]:
+    return models if fit_filter is None else tuple(model for model in models if model in fit_filter)
+
+
+def _apply_context_placement(
+    tier: ComplexityTier | str, signals: tuple[str, ...], placement: _ContextWindowPlacement | None
+) -> tuple[ComplexityTier | str, tuple[str, ...], ComplexityTier | str | None]:
+    """(final tier, signals, original tier when the gate escalated, else None)."""
+    if placement is None:
+        return tier, signals, None
+    if _tier_name(placement.tier) == _tier_name(tier):
+        return placement.tier, signals, None
+    return placement.tier, (*signals, "context_escalation"), tier
+
+
+def _window_can_hold(window: int | None, needed: int, buffer: float) -> bool:
+    return window is None or needed <= int(window * buffer)
+
+
+def _group_provably_fits(facts: tuple[int | None, bool], needed: int, buffer: float) -> bool:
+    window, has_unknown = facts
+    return window is not None and not has_unknown and needed <= int(window * buffer)
+
+
+class _ContextWindowPlacement(NamedTuple):
+    """Where the context-window gate placed the request: the placement tier, the subset of its
+    pool the pick may use, and every configured group not provably misfit (the adaptive filter)."""
+
+    tier: ComplexityTier | str
+    allowed_models: tuple[str, ...]
+    holdable_models: frozenset[str]
+
+
 class _SessionAffinityPin(NamedTuple):
     model: str
     tier: ComplexityTier | None
@@ -1195,6 +1264,7 @@ class ComplexityRouter(CustomLogger):
         classifier_cost: float | None = None,
         conversation_continuing: bool = True,
         tier_litellm_params: Mapping[str, object] | None = None,
+        context_escalation_original_tier: ComplexityTier | str | None = None,
     ) -> StandardLoggingRoutingDecision:
         """Assemble the per-request provenance record for this router's decision.
 
@@ -1244,6 +1314,12 @@ class ComplexityRouter(CustomLogger):
             decision["classifier_model"] = classifier_model
         if classifier_cost is not None:
             decision["classifier_cost"] = classifier_cost
+        if context_escalation_original_tier is not None:
+            # The pair travels together: the flag says the gate moved the request off its
+            # decided tier on prompt size, and the original tier names where the decision
+            # (classifier, keyword rule, or session pin) had placed it before physics did.
+            decision["context_escalated"] = True
+            decision["context_escalation_original_tier"] = _tier_name(context_escalation_original_tier)
         if tier_litellm_params:
             masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params)
             if isinstance(masked_tier_litellm_params, Mapping):
@@ -1644,7 +1720,7 @@ class ComplexityRouter(CustomLogger):
         return entry.litellm_params if entry is not None else MappingProxyType({})
 
     @staticmethod
-    def _pick_from_tier_value(model: str | list[str], tier_key: str) -> str:
+    def _pick_from_tier_value(model: str | Sequence[str], tier_key: str) -> str:
         if isinstance(model, str):
             return model
         if not model:
@@ -1660,15 +1736,21 @@ class ComplexityRouter(CustomLogger):
         raw_messages: list[dict[str, Any]] | None,
         resolved_messages: list[dict[str, Any]] | None,
         request_kwargs: dict,
+        allowed_models: tuple[str, ...] | None = None,
     ) -> str:
         if not self.config.plugins:
+            if allowed_models is not None:
+                return self._pick_from_tier_value(allowed_models, _tier_name(tier))
             return self.get_model_for_tier(tier)
 
         from litellm.types.router import RoutingContext
 
         tier_key: Final = _tier_name(tier)
         metadata_key: Final = get_metadata_variable_name_from_kwargs(request_kwargs)
-        pool: Final = tuple(self._tier_pools().get(tier_key, ()))
+        full_pool: Final = tuple(self._tier_pools().get(tier_key, ()))
+        pool: Final = (
+            tuple(model for model in full_pool if model in allowed_models) if allowed_models is not None else full_pool
+        )
         if not pool:
             # Nothing for the plugins to filter. Falling through would raise the
             # plugin-filtering error below and send the operator hunting for a policy
@@ -1762,6 +1844,7 @@ class ComplexityRouter(CustomLogger):
         request_kwargs: dict[str, Any] | None = None,
         hard_floor: ComplexityTier | str | None = None,
         hard_ceiling: ComplexityTier | str | None = None,
+        fit_filter: frozenset[str] | None = None,
     ) -> str:
         """hard_floor excludes every candidate whose tiers all sit below it, turning this pick's
         soft floors (a distance penalty a high-scoring cheap model can outweigh) into a hard
@@ -1774,7 +1857,10 @@ class ComplexityRouter(CustomLogger):
         tier because that is all it is worth, so a bandit trading cost for quality has nothing to
         win and must not reach above it. Without it the distance penalty is the only thing holding
         the tier, and a deployment that lowers tier_distance_penalty silently gets the expensive
-        model back while the routing decision still reads as the cheapest tier."""
+        model back while the routing decision still reads as the cheapest tier.
+
+        fit_filter excludes candidates the context-window gate proved cannot hold the prompt,
+        in every phase including cold start and the tier fallbacks."""
         from litellm.router_strategy.adaptive_router.bandit import (
             normalized_cost,
             thompson_sample,
@@ -1785,12 +1871,12 @@ class ComplexityRouter(CustomLogger):
         if adaptive is None or not isinstance(classified_tier, ComplexityTier):
             # Custom tier names have no severity index; adaptive is rejected alongside
             # tier_definitions, so this guard is the contract for any future caller.
-            return self.get_model_for_tier(classified_tier)
+            return self._fitting_tier_fallback(classified_tier, fit_filter)
 
         request_type: Final = classify_prompt(user_message)
         classified_idx: Final = TIER_SEVERITY_ORDER.index(classified_tier)
         pools: Final = self._tier_pools()
-        classified_candidates: Final = tuple(pools.get(_tier_name(classified_tier), ()))
+        classified_candidates: Final = _allowed(tuple(pools.get(_tier_name(classified_tier), ())), fit_filter)
         cold_start_candidates: Final = tuple(
             model for model in classified_candidates if adaptive._cells[(request_type, model)].total_samples == 0
         )
@@ -1820,9 +1906,9 @@ class ComplexityRouter(CustomLogger):
         if self.config.adaptive_eligible == "classified_tier":
             candidates = list(classified_candidates)
             if not candidates:
-                return self.get_model_for_tier(classified_tier)
+                return self._fitting_tier_fallback(classified_tier, fit_filter)
         else:
-            candidates = list(adaptive.config.available_models)
+            candidates = list(_allowed(tuple(adaptive.config.available_models), fit_filter))
 
         all_costs: Final = [adaptive.model_to_cost.get(m, 0.0) for m in candidates]
         quality_weight: Final = self.config.adaptive_weights.quality
@@ -1869,7 +1955,7 @@ class ComplexityRouter(CustomLogger):
                 best_score = score
                 best_model = model
         if best_model is None:
-            return self.get_model_for_tier(classified_tier)
+            return self._fitting_tier_fallback(classified_tier, fit_filter)
         if request_kwargs is not None:
             metadata = request_kwargs.setdefault("metadata", {})
             if isinstance(metadata, dict):
@@ -1886,6 +1972,12 @@ class ComplexityRouter(CustomLogger):
                 }
         return best_model
 
+    def _fitting_tier_fallback(self, classified_tier: ComplexityTier | str, fit_filter: frozenset[str] | None) -> str:
+        fitting: Final = _allowed(tuple(self._tier_pools().get(_tier_name(classified_tier), ())), fit_filter)
+        if fit_filter is not None and fitting:
+            return self._pick_from_tier_value(fitting, _tier_name(classified_tier))
+        return self.get_model_for_tier(classified_tier)
+
     def _resolve_plan_mode_floor(self) -> ComplexityTier | str | None:
         """The configured floor as an active tier: the built-in enum member, or the defined
         name itself for a custom tier set; None when the feature is off."""
@@ -1956,6 +2048,163 @@ class ComplexityRouter(CustomLogger):
             return None
         return name if self.config.has_custom_tiers else ComplexityTier(name)
 
+    def _deployment_window(self, group: str, deployment: Mapping[str, object]) -> int | None:
+        from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
+
+        deployment_model_info: Final = deployment.get("model_info")
+        declared: Final = (
+            deployment_model_info.get("max_input_tokens") if isinstance(deployment_model_info, Mapping) else None
+        )
+        if isinstance(declared, int):
+            return declared
+        litellm_params: Final = deployment.get("litellm_params")
+        params: Final = litellm_params if isinstance(litellm_params, Mapping) else EMPTY_MAPPING
+        provider_override: Final = params.get("custom_llm_provider")
+        # get_router_model_info resolves the provider, and get_llm_provider runs the OAuth device
+        # flow for github_copilot/chatgpt, so a metadata question must never reach it for those.
+        if declared_authenticating_provider(
+            str(params.get("model") or ""), provider_override if isinstance(provider_override, str) else None
+        ):
+            return None
+        try:
+            model_info: Final = self.litellm_router_instance.get_router_model_info(
+                deployment=cast(dict, deployment),  # cast-ok: router deployments are plain dicts
+                received_model_name=group,
+            )
+            window: Final = model_info.get("max_input_tokens")
+        except Exception:  # noqa: BLE001  # best-effort: an unmappable deployment must not hide the others
+            return None
+        return window if isinstance(window, int) else None
+
+    def _group_window_facts(self, group: str) -> tuple[int | None, bool]:
+        """(smallest declared context window across the group's deployments, whether any deployment
+        declares none). The core router picks a deployment within the group without a fit check, so
+        the group is only as safe as its smallest member."""
+        list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None)
+        deployments: Final = list_models(model_name=group) if callable(list_models) else None
+        if not isinstance(deployments, list) or not deployments:
+            return (None, True)
+        windows: Final = tuple(
+            window for deployment in deployments if (window := self._deployment_window(group, deployment)) is not None
+        )
+        return (min(windows) if windows else None, len(windows) < len(deployments))
+
+    @staticmethod
+    def _out_of_band_request_text(request_kwargs: Mapping[str, object]) -> str:
+        """Prompt content the resolved message list never carries: the Responses API's
+        `instructions`, the /v1/messages top-level `system` block, and tool definitions.
+        A coding agent's context is dominated by these."""
+        import json
+
+        instructions: Final = request_kwargs.get("instructions")
+        proxy_request: Final = request_kwargs.get("proxy_server_request")
+        body: Final = proxy_request.get("body") if isinstance(proxy_request, Mapping) else None
+        system: Final = body.get("system") if isinstance(body, Mapping) else None
+        tools: Final = (
+            body.get("tools") if isinstance(body, Mapping) and body.get("tools") else request_kwargs.get("tools")
+        )
+        tools_text = ""
+        if tools:
+            try:
+                tools_text = json.dumps(tools, default=str)
+            except (TypeError, ValueError):
+                tools_text = str(tools)
+        return (
+            (instructions if isinstance(instructions, str) else "")
+            + (str(system) if system is not None else "")
+            + tools_text
+        )
+
+    def _request_byte_upper_bound(
+        self, resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: Mapping[str, object]
+    ) -> int:
+        """UTF-8 byte length of all prompt content. BPE emits at least one byte per token in every
+        script, so the token count never exceeds this and 'bytes fit' soundly skips counting."""
+        content_bytes: Final = sum(len(str(m.get("content") or "").encode()) for m in resolved_messages or ())
+        return content_bytes + len(self._out_of_band_request_text(request_kwargs).encode())
+
+    async def _counted_request_tokens(
+        self, resolved_messages: Sequence[Mapping[str, object]], request_kwargs: Mapping[str, object]
+    ) -> int | None:
+        """Real-tokenizer count of the resolved messages plus the out-of-band carriers, off the
+        event loop; None when counting fails, and the gate then leaves the placement alone."""
+        import litellm
+        from litellm.litellm_core_utils.asyncify import asyncify
+
+        out_of_band: Final = self._out_of_band_request_text(request_kwargs)
+        try:
+            counted: Final = await asyncify(litellm.token_counter)(
+                messages=cast(list, resolved_messages)  # cast-ok: token_counter only iterates the sequence
+            )
+            return counted + (await asyncify(litellm.token_counter)(text=out_of_band) if out_of_band else 0)
+        except Exception as e:  # noqa: BLE001  # best-effort: an uncountable prompt must not fail the request
+            verbose_router_logger.debug("ComplexityRouter: context-window token count failed. Got - %s", e)
+            return None
+
+    async def _context_window_placement(
+        self,
+        tier: ComplexityTier | str,
+        resolved_messages: Sequence[Mapping[str, object]] | None,
+        request_kwargs: Mapping[str, object],
+        pool_override: tuple[str, ...] | None = None,
+    ) -> _ContextWindowPlacement | None:
+        """Correct a decided placement whose models provably cannot hold the prompt, or None
+        (the placement stands). Only a real tokenizer count ever moves a request, escalation
+        lands only on groups whose every deployment declares a fitting window, and a group
+        with no resolvable window is never moved on faith in either direction."""
+        if not self.config.enable_context_window_escalation or not resolved_messages:
+            return None
+        pools: Final = self._tier_pools()
+        pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ()))
+        if not pool:
+            return None
+        facts: Final = MappingProxyType({group: self._group_window_facts(group) for group in pool})
+        known_windows: Final = tuple(window for window, _ in facts.values() if window is not None)
+        if not known_windows:
+            return None
+        buffer: Final = self.config.context_window_escalation_buffer
+        if self._request_byte_upper_bound(resolved_messages, request_kwargs) <= int(min(known_windows) * buffer):
+            return None
+        needed: Final = await self._counted_request_tokens(resolved_messages, request_kwargs)
+        if needed is None:
+            return None
+        return self._placement_for_tokens(tier=tier, pool=pool, pools=pools, facts=facts, needed=needed)
+
+    def _placement_for_tokens(
+        self,
+        *,
+        tier: ComplexityTier | str,
+        pool: tuple[str, ...],
+        pools: Mapping[str, list[str]],
+        facts: Mapping[str, tuple[int | None, bool]],
+        needed: int,
+    ) -> _ContextWindowPlacement | None:
+        buffer: Final = self.config.context_window_escalation_buffer
+        in_tier: Final = tuple(group for group in pool if _window_can_hold(facts[group][0], needed, buffer))
+        if in_tier and len(in_tier) == len(pool):
+            return None
+        holdable: Final = frozenset(
+            group
+            for tier_pool in pools.values()
+            for group in tier_pool
+            if _window_can_hold(self._group_window_facts(group)[0], needed, buffer)
+        )
+        if in_tier:
+            return _ContextWindowPlacement(tier=tier, allowed_models=in_tier, holdable_models=holdable)
+        for name in self.config.tier_names()[self._active_tier_severity(tier) + 1 :]:
+            proven = tuple(
+                group
+                for group in pools.get(name, ())
+                if _group_provably_fits(self._group_window_facts(group), needed, buffer)
+            )
+            if proven:
+                return _ContextWindowPlacement(
+                    tier=name if self.config.has_custom_tiers else ComplexityTier(name),
+                    allowed_models=proven,
+                    holdable_models=holdable,
+                )
+        return None
+
     def _apply_plan_mode_floor(self, tier: ComplexityTier | str) -> ComplexityTier | str:
         """The higher of the decided tier and the plan-mode floor; identity when the floor is unset."""
         floor: Final = self._resolve_plan_mode_floor()
@@ -2247,14 +2496,18 @@ class ComplexityRouter(CustomLogger):
 
     @property
     def _uses_tier_pin(self) -> bool:
-        return bool(self.config.session_affinity and not self.config.plugins)
+        """classification_mode 'user_turn' implies the tier pin machinery: the pin write after each
+        pinnable classification is what gives a continuation a held decision to replay."""
+        return bool(
+            (self.config.session_affinity or self.config.classification_mode == "user_turn") and not self.config.plugins
+        )
 
     @property
     def _uses_deployment_pin(self) -> bool:
-        """session_affinity implies the deployment pin: a session frozen onto one model
+        """The tier pin implies the deployment pin: a session frozen onto one model
         group but load-balanced across its deployments would still go cache-cold, which
         is the exact failure both flags exist to prevent."""
-        return bool((self.config.deployment_affinity or self.config.session_affinity) and not self.config.plugins)
+        return bool(self.config.deployment_affinity and not self.config.plugins) or self._uses_tier_pin
 
     def _with_session_deployment_affinity(
         self, response: PreRoutingHookResponse | None
@@ -2282,6 +2535,11 @@ class ComplexityRouter(CustomLogger):
         pins the model chosen on the session's first turn and reuses it for every later
         turn, skipping classification entirely. Otherwise delegates to `_classify_and_route`.
 
+        When `classification_mode` is 'user_turn', the same pin is replayed only on
+        continuation turns (an agent loop's tool traffic); a new human ask always falls
+        through to classification, so the session can still move tiers between asks.
+        With both knobs on, session_affinity's pin-first behavior wins.
+
         Skipped entirely when `plugins` are configured: reusing a stale pin would bypass
         the plugin pipeline on every turn after the first, since a pinned model was never
         re-checked against a policy plugin whose decision can change between turns (e.g. a
@@ -2305,7 +2563,13 @@ class ComplexityRouter(CustomLogger):
         session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None
         cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None
 
-        if cache_key is not None:
+        # In 'user_turn' mode a held pin is replayed only on continuation turns; a new human
+        # ask falls through and re-classifies. session_affinity restores pin-first for asks too.
+        pin_replay_allowed: Final = bool(self.config.session_affinity) or not _newest_turn_is_human_ask(
+            resolved_messages, self._reminder_markers
+        )
+
+        if cache_key is not None and pin_replay_allowed:
             pinned_value: Final = await self.litellm_router_instance.cache.async_get_cache(key=cache_key)
             pinned_pin: Final = _parse_session_affinity_pin(pinned_value)
             if pinned_pin is not None:
@@ -2339,6 +2603,26 @@ class ComplexityRouter(CustomLogger):
                     session_model: Final = routed_model
                     if plan_floored and pinned_tier is not None:
                         routed_model = self.get_model_for_tier(self._apply_plan_mode_floor(pinned_tier))
+                    pin_source_tier: Final = self._tier_for_model(routed_model)
+                    pin_placement: Final = (
+                        await self._context_window_placement(
+                            pin_source_tier, resolved_messages, request_kwargs, pool_override=(routed_model,)
+                        )
+                        if pin_source_tier is not None
+                        else None
+                    )
+                    pin_context_original_tier: Final = (
+                        pin_source_tier
+                        if pin_placement is not None
+                        and pin_source_tier is not None
+                        and _tier_name(pin_placement.tier) != _tier_name(pin_source_tier)
+                        else None
+                    )
+                    if pin_placement is not None and pin_context_original_tier is not None:
+                        # The stored pin below keeps the session's own model on purpose.
+                        routed_model = self._pick_from_tier_value(
+                            pin_placement.allowed_models, _tier_name(pin_placement.tier)
+                        )
                     # Refresh the TTL on every hit so an active session doesn't lose its
                     # pin mid-conversation just because it outlives the original write.
                     await self.litellm_router_instance.cache.async_set_cache(
@@ -2354,15 +2638,20 @@ class ComplexityRouter(CustomLogger):
                         kwargs_metadata: Final = request_kwargs.setdefault("metadata", {})
                         if isinstance(kwargs_metadata, dict):
                             kwargs_metadata[ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY] = routed_model
+                    replay_cause: Final[RoutingDecisionCause] = (
+                        "session_affinity_pin" if self.config.session_affinity else "user_turn_continuation"
+                    )
                     cause: RoutingDecisionCause = (
-                        "plan_mode"
-                        if plan_floored
-                        else ("session_affinity_escalation" if escalated else "session_affinity_pin")
+                        "plan_mode" if plan_floored else ("session_affinity_escalation" if escalated else replay_cause)
                     )
                     verbose_router_logger.info(
                         "ComplexityRouter: routing decision cause=%s, routed_model=%s", cause, routed_model
                     )
-                    routed_pin_tier: Final = self._tier_for_model(routed_model) if plan_floored else resolved_pin_tier
+                    routed_pin_tier: Final = (
+                        pin_placement.tier
+                        if pin_placement is not None and pin_context_original_tier is not None
+                        else (self._tier_for_model(routed_model) if plan_floored else resolved_pin_tier)
+                    )
                     session_tier_litellm_params: Final = self._litellm_params_for_model(routed_pin_tier, routed_model)
                     has_original_messages: Final = messages is not None and len(messages) > 0
                     return self._with_session_deployment_affinity(
@@ -2379,6 +2668,7 @@ class ComplexityRouter(CustomLogger):
                                 escalated=escalated,
                                 conversation_continuing=conversation_continuing,
                                 tier_litellm_params=session_tier_litellm_params,
+                                context_escalation_original_tier=pin_context_original_tier,
                             ),
                         )
                     )
@@ -2573,6 +2863,8 @@ class ComplexityRouter(CustomLogger):
         plan_floored: Final = tier != pre_floor_tier
         if plan_floored:
             signals = (*signals, "plan_mode_floor")
+        context_placement: Final = await self._context_window_placement(tier, resolved_messages, request_kwargs)
+        tier, signals, context_original_tier = _apply_context_placement(tier, signals, context_placement)
         score_repr: Final = f"{score:.3f}" if score is not None else "n/a"
         fallback_model: Final = self.config.default_model if not self.config.plugins else None
         # A sentinel-carrying request skips the failure exit below, whether or not the floor
@@ -2619,8 +2911,15 @@ class ComplexityRouter(CustomLogger):
             # the cheapest tier would then contradict the floor and bound the pick below the tier
             # the decision reports.
             housekeeping_ceiling: Final = tier if outcome.cause == "housekeeping" else None
+            # A context-escalated tier becomes the hard floor: a floor the bandit can slide
+            # under is not a floor.
             routed_model = self._soft_floor_pick(
-                tier, user_message, request_kwargs, hard_floor=plan_floor, hard_ceiling=housekeeping_ceiling
+                tier,
+                user_message,
+                request_kwargs,
+                hard_floor=tier if context_original_tier is not None else plan_floor,
+                hard_ceiling=housekeeping_ceiling,
+                fit_filter=context_placement.holdable_models if context_placement is not None else None,
             )
             adaptive: Final = self._ensure_adaptive_router()
             if adaptive is not None:
@@ -2637,7 +2936,13 @@ class ComplexityRouter(CustomLogger):
                 routed_model,
             )
         else:
-            routed_model = await self._pick_model_for_tier(tier, messages, resolved_messages, request_kwargs)
+            routed_model = await self._pick_model_for_tier(
+                tier,
+                messages,
+                resolved_messages,
+                request_kwargs,
+                allowed_models=context_placement.allowed_models if context_placement is not None else None,
+            )
             verbose_router_logger.info(
                 "ComplexityRouter: routing decision cause=%s, tier=%s, score=%s, signals=%s, routed_model=%s",
                 outcome.cause,
@@ -2690,5 +2995,6 @@ class ComplexityRouter(CustomLogger):
                 classifier_model=classifier_model,
                 classifier_cost=outcome.classifier_cost,
                 tier_litellm_params=tier_litellm_params,
+                context_escalation_original_tier=context_original_tier,
             ),
         )

litellm/router_strategy/complexity_router/config.py

@@ -823,6 +823,32 @@ class ComplexityRouterConfig(BaseModel):
         ),
     )
 
+    enable_context_window_escalation: bool = Field(
+        default=True,
+        description=(
+            "Escalate a request off a tier whose models provably cannot hold its prompt, before "
+            "dispatch. The classifier scores complexity and never prompt size, so a long agentic "
+            "session whose newest ask is trivial lands on a small-window tier and the provider "
+            "rejects it with a context-window 400 that nothing retries. When every model of the "
+            "decided tier has a declared window smaller than the estimated prompt, the request "
+            "moves to the lowest configured tier with a model whose declared window fits; when "
+            "only some of the tier's models fit, the pick is restricted to those and the tier "
+            "keeps the request. Models with no resolvable window are never escalated away from "
+            "and never escalated onto. Set false to dispatch on complexity alone, as before."
+        ),
+    )
+    context_window_escalation_buffer: float = Field(
+        default=0.95,
+        gt=0,
+        le=1,
+        description=(
+            "Fraction of a model's declared context window the estimated prompt must fit within. "
+            "The token count is an estimate, so fitting against the full window would dispatch "
+            "prompts that the provider's own tokenizer then rejects; 0.95 leaves room for that "
+            "drift plus the response tokens."
+        ),
+    )
+
     # Semantic (embedding) matching for keyword_tier_rules instead of literal text matching
     semantic_keyword_matching: bool = Field(
         default=False,
@@ -839,6 +865,21 @@ class ComplexityRouterConfig(BaseModel):
         description="Minimum cosine similarity for a semantic keyword match",
     )
 
+    classification_mode: Literal["every_request", "user_turn"] = Field(
+        default="every_request",
+        description=(
+            "When to run the complexity classifier. 'every_request' (the default) classifies every "
+            "inference request, including the tool-result continuation turns of an agentic loop. "
+            "'user_turn' classifies only requests whose newest turn is a new human ask and replays "
+            "the session's held routing decision on continuation turns, which cuts classifier "
+            "spend and eliminates mid-loop model switches. Continuations with no held decision to "
+            "replay (no resolvable session_id, expired pin, fresh restart) still classify. Unlike "
+            "session_affinity, a new human ask always re-classifies, so a session can still move "
+            "tiers between asks. Suppressed when plugins are configured, for the same reason "
+            "session_affinity is: a replayed decision would bypass the plugin pipeline."
+        ),
+    )
+
     # Session affinity: pin the first turn's routed model for the rest of the session
     session_affinity: bool = Field(
         default=False,

litellm/types/llms/openai.py

@@ -2162,6 +2162,42 @@ class OpenAIRealtimeDoneEvent(TypedDict):
     type: Literal["response.done"]
 
 
+class OpenAIRealtimeInputAudioBufferSpeechEvent(TypedDict):
+    type: ReadOnly[Literal["input_audio_buffer.speech_started", "input_audio_buffer.speech_stopped"]]
+    event_id: ReadOnly[str]
+    item_id: ReadOnly[str]
+
+
+class OpenAIRealtimeInputAudioTranscriptionDelta(TypedDict):
+    type: ReadOnly[Literal["conversation.item.input_audio_transcription.delta"]]
+    event_id: ReadOnly[str]
+    item_id: ReadOnly[str]
+    content_index: ReadOnly[int]
+    delta: ReadOnly[str]
+
+
+class OpenAIRealtimeInputAudioTranscriptionCompleted(TypedDict):
+    type: ReadOnly[Literal["conversation.item.input_audio_transcription.completed"]]
+    event_id: ReadOnly[str]
+    item_id: ReadOnly[str]
+    content_index: ReadOnly[int]
+    transcript: ReadOnly[str]
+
+
+class OpenAIRealtimeUsageTokenDetails(TypedDict):
+    audio_tokens: ReadOnly[int]
+    text_tokens: ReadOnly[int]
+    cached_tokens: NotRequired[ReadOnly[int]]
+
+
+class OpenAIRealtimeResponseUsage(TypedDict):
+    input_tokens: ReadOnly[int]
+    output_tokens: ReadOnly[int]
+    total_tokens: ReadOnly[int]
+    input_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]]
+    output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]]
+
+
 class OpenAIRealtimeEventTypes(Enum):
     SESSION_CREATED = "session.created"
     # Beta delta event names
@@ -2199,6 +2235,9 @@ OpenAIRealtimeEvents = (
     | OpenAIRealtimeOutputItemDone
     | OpenAIRealtimeFunctionCallArgumentsDone
     | OpenAIRealtimeDoneEvent
+    | OpenAIRealtimeInputAudioBufferSpeechEvent
+    | OpenAIRealtimeInputAudioTranscriptionDelta
+    | OpenAIRealtimeInputAudioTranscriptionCompleted
 )
 
 OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents]

litellm/types/management_endpoints/auto_router_endpoints.py

@@ -245,6 +245,8 @@ ShadowEvalStatus: TypeAlias = Literal["running", "completed", "stopped"]
 
 ShadowEvalDirection: TypeAlias = Literal["forward", "reverse"]
 
+ShadowEvalTargetType: TypeAlias = Literal["key", "team", "user"]
+
 DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5"
 
 # Sample-count ceiling written on every new job: a zero-cost error loop (a shadow arm that
@@ -253,16 +255,37 @@ SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000
 
 
 class StartShadowEvalRequest(BaseModel):
-    """Start duplicating one or more keys' traffic for blind comparison against an auto-router."""
+    """Start duplicating one or more targets' traffic for blind comparison against an auto-router.
+
+    A target is a virtual key, a team, or a user; each becomes its own leg with its own
+    budget and stop state. Team and user targets match on the identity every request
+    carries after auth (user_api_key_team_id / user_api_key_user_id), so they cover
+    JWT-authenticated traffic, which presents no virtual key at all."""
 
     api_key_ids: tuple[str, ...] = Field(
-        min_length=1,
+        default=(),
         max_length=100,
         description=(
-            "The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these "
-            "keys' traffic; requests made with any other key are not sampled. Each key carries its own "
-            "max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 "
-            "keys per job, which also bounds every read the job's endpoints make."
+            "Hashed virtual keys whose traffic will be shadowed. Combined with team_ids and user_ids the job "
+            "needs at least one target and at most 100, which also bounds every read the job's endpoints make. "
+            "Each target carries its own max_budget spend budget, so one exhausting its budget leaves the "
+            "others sampling."
+        ),
+    )
+    team_ids: tuple[str, ...] = Field(
+        default=(),
+        max_length=100,
+        description=(
+            "Teams whose traffic will be shadowed, matched on the team every authenticated request resolves "
+            "to, so a team's JWT-auth and virtual-key traffic are both sampled"
+        ),
+    )
+    user_ids: tuple[str, ...] = Field(
+        default=(),
+        max_length=100,
+        description=(
+            "Users whose traffic will be shadowed, matched on the user every authenticated request resolves "
+            "to across all their teams: JWT requests carrying their subject claim and virtual keys they own"
         ),
     )
     router_name: str = Field(description="The auto-router under evaluation, in either direction")
@@ -285,7 +308,7 @@ class StartShadowEvalRequest(BaseModel):
     shadow_percentage: float = Field(
         ge=0.1,
         le=100.0,
-        description="Percentage of the key's requests to duplicate through the router",
+        description="Percentage of each target's requests to duplicate through the router",
     )
     judge_model: str = Field(
         default=DEFAULT_SHADOW_EVAL_JUDGE_MODEL,
@@ -306,9 +329,9 @@ class StartShadowEvalRequest(BaseModel):
         ge=0.01,
         le=10_000,
         description=(
-            "Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with "
-            "the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval "
-            "spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight "
+            "Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with "
+            "the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval "
+            "spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight "
             "samples can overshoot the cap by one sampling cache window"
         ),
     )
@@ -319,7 +342,7 @@ class StartShadowEvalRequest(BaseModel):
         """Pydantic ignores unknown fields, so a caller still sending max_turns would
         silently run on the default dollar budget instead of the bound they asked for."""
         if isinstance(values, Mapping) and "max_turns" in values:
-            raise ValueError("max_turns was replaced by max_budget, the per-key USD cap on the eval's own spend")
+            raise ValueError("max_turns was replaced by max_budget, the per-target USD cap on the eval's own spend")
         return values
 
     @field_validator("shadow_percentage")
@@ -327,12 +350,21 @@ class StartShadowEvalRequest(BaseModel):
     def _round_percentage(cls, value: float) -> float:
         return round(value, 2)
 
-    @field_validator("api_key_ids")
+    @field_validator("api_key_ids", "team_ids", "user_ids")
     @classmethod
-    def _dedupe_keys(cls, value: tuple[str, ...]) -> tuple[str, ...]:
-        """A key named twice would collide with itself on the one-active-per-(key, direction) index."""
+    def _dedupe_targets(cls, value: tuple[str, ...]) -> tuple[str, ...]:
+        """A target named twice would collide with itself on the one-active-per-(target, direction) index."""
         return tuple(dict.fromkeys(value))
 
+    @model_validator(mode="after")
+    def _at_least_one_target_at_most_hundred(self) -> "StartShadowEvalRequest":
+        total: Final = len(self.api_key_ids) + len(self.team_ids) + len(self.user_ids)
+        if total < 1:
+            raise ValueError("at least one target is required: pass api_key_ids, team_ids, or user_ids")
+        if total > 100:
+            raise ValueError("at most 100 targets per job across api_key_ids, team_ids, and user_ids")
+        return self
+
     @model_validator(mode="after")
     def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest":
         if self.direction == "reverse" and self.baseline_model is None:
@@ -343,8 +375,9 @@ class StartShadowEvalRequest(BaseModel):
 
 
 class ShadowEvalSlice(BaseModel):
-    """Judge outcomes for one slice of a job's verdicts (a router tier, or one of the
-    models that served the real arm)."""
+    """Judge outcomes for one slice of a job's verdicts: a router tier, one of the
+    models that served the real arm, or one scoped target (embedded on that target's
+    own entry, so slices never need re-joining to a target by id)."""
 
     group: str
     turn_count: int
@@ -395,12 +428,6 @@ class ShadowEvalResult(BaseModel):
             "and in reverse the models the router itself picked"
         )
     )
-    by_key: tuple[ShadowEvalSlice, ...] = Field(
-        description=(
-            "One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job "
-            "scopes but has not judged a turn for yet are absent rather than reported as zero"
-        ),
-    )
     overall_shadow_win_rate_pct: float
     overall_tie_rate_pct: float
     sampled_real_spend: float = Field(
@@ -436,27 +463,28 @@ class ShadowEvalResult(BaseModel):
     )
 
 
-class ShadowEvalJobKeyResponse(BaseModel):
-    """One key a job shadows, with its own budget and stop state."""
+class ShadowEvalJobTargetResponse(BaseModel):
+    """One target a job shadows (a key, team, or user), with its own budget and stop state."""
 
-    api_key_id: str = Field(description="The hashed virtual key whose traffic this entry scopes")
+    target_type: ShadowEvalTargetType = Field(description="What kind of entity this entry scopes")
+    target_id: str = Field(description="The hashed virtual key, team id, or user id whose traffic this entry scopes")
     max_turns: int = Field(
         description=(
-            "This key's sample-count ceiling: the whole budget for jobs created before max_budget "
+            "This target's sample-count ceiling: the whole budget for jobs created before max_budget "
             "existed, and the error-loop safety valve otherwise"
         )
     )
     max_budget: float | None = Field(
         default=None,
         description=(
-            "This key's own USD budget for the eval's shadow and judge spend, independent of its "
+            "This target's own USD budget for the eval's shadow and judge spend, independent of its "
             "siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds"
         ),
     )
     stopped_at: datetime | None = Field(
         default=None,
         description=(
-            "When this key's slot was stamped free, whether its own budget ran out, the window closed, "
+            "When this target's slot was stamped free, whether its own budget ran out, the window closed, "
             "or an operator stopped the job; status is derived, so a spent budget reads completed even "
             "while this is still unset"
         ),
@@ -464,45 +492,53 @@ class ShadowEvalJobKeyResponse(BaseModel):
     attempt_count: int | None = Field(
         default=None,
         description=(
-            "This key's sampled attempts so far, judged and errored alike, the same count the sampler "
+            "This target's sampled attempts so far, judged and errored alike, the same count the sampler "
             "budgets against max_turns; populated on list and detail responses. Frozen at stopped_at "
-            "once the key is stamped, so in-flight attempts landing after a stop never reclassify it"
+            "once the target is stamped, so in-flight attempts landing after a stop never reclassify it"
         ),
     )
     spend: float | None = Field(
         default=None,
         description=(
-            "This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets "
+            "This target's recorded shadow plus judge spend in USD, the same figure the sampler budgets "
             "against max_budget; populated on list and detail responses and frozen at stopped_at "
             "exactly like attempt_count"
         ),
     )
 
+    verdicts: "ShadowEvalSlice | None" = Field(
+        default=None,
+        description="This target's own judged-verdict slice; detail endpoint only, None until a turn is judged",
+    )
+
     @property
     def budget_spent(self) -> bool:
         over_spend: Final = self.max_budget is not None and self.spend is not None and self.spend >= self.max_budget
         return over_spend or (self.attempt_count is not None and self.attempt_count >= self.max_turns)
 
-    key_alias: str | None = Field(
+    target_alias: str | None = Field(
         default=None,
-        description="Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted",
+        description=(
+            "Display label resolved from the target's own row at read time: the key's alias, the team's "
+            "alias, or the user's email; None when unset or deleted"
+        ),
     )
     key_name: str | None = Field(
         default=None,
-        description="Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias",
+        description="Masked display name (sk-...) for key targets, resolved at read time; None for teams and users",
     )
 
 
 class ShadowEvalJobResponse(BaseModel):
-    """A shadow-eval job over one or more keys, each with its own budget and stop state;
-    status is derived from stopped_by, the keys' stop and budget state, and ends_at,
+    """A shadow-eval job over one or more targets, each with its own budget and stop state;
+    status is derived from stopped_by, the targets' stop and budget state, and ends_at,
     never stored, so no writer anywhere can produce an inconsistent one. Aggregate
     fields are populated by the detail endpoint only and stay None on list responses."""
 
     job_id: str
-    keys: tuple[ShadowEvalJobKeyResponse, ...] = Field(
+    targets: tuple[ShadowEvalJobTargetResponse, ...] = Field(
         min_length=1,
-        description="The keys whose traffic this job evaluates, and only those keys', each with its own budget",
+        description="The targets whose traffic this job evaluates, and only theirs, each with its own budget",
     )
     router_name: str
     direction: ShadowEvalDirection = "forward"
@@ -531,8 +567,8 @@ class ShadowEvalJobResponse(BaseModel):
     def status(self) -> ShadowEvalStatus:
         """Three recorded facts, no history-guessing: a stop is stopped_by (the migration
         backfills it for every job that displayed stopped when the column arrived, so the
-        pre-column population is closed), completion is the window passing or every key
-        spending its budget, and anything else is running. The all-keys-stamped fallback
+        pre-column population is closed), completion is the window passing or every target
+        spending its budget, and anything else is running. The all-targets-stamped fallback
         covers only stops written by pre-column pods during a rolling deploy."""
         if self.stopped_by is not None:
             return "stopped"
@@ -540,8 +576,8 @@ class ShadowEvalJobResponse(BaseModel):
             self.ends_at if self.ends_at.tzinfo else self.ends_at.replace(tzinfo=timezone.utc)
         ):
             return "completed"
-        if all(key.budget_spent for key in self.keys):
+        if all(target.budget_spent for target in self.targets):
             return "completed"
-        if all(key.stopped_at is not None for key in self.keys):
+        if all(target.stopped_at is not None for target in self.targets):
             return "stopped"
         return "running"

litellm/types/utils.py

@@ -2842,6 +2842,11 @@ RoutingDecisionCause = Literal[
     "housekeeping",
     "session_affinity_pin",
     "session_affinity_escalation",
+    # classification_mode 'user_turn': the request is an agent loop's continuation turn (no new
+    # human ask), so the session's held routing decision was replayed and the classifier was never
+    # called. Distinct from "session_affinity_pin", which reports the session_affinity flag pinning
+    # every turn including new asks; this cause only appears when session_affinity is off.
+    "user_turn_continuation",
     "default_fallback",
     "keyword",
     "quality_tier",
@@ -2881,6 +2886,8 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
     classifier_model: str
     classifier_cost: float
     escalated: bool
+    context_escalated: bool  # writable-ok: Pydantic warns on ReadOnly TypedDict fields
+    context_escalation_original_tier: str  # writable-ok: Pydantic warns on ReadOnly TypedDict fields
     tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries
     reasoning_override_min_score: float  # writable-ok: Pydantic warns on ReadOnly TypedDict fields
     conversation_continuing: bool
@@ -2907,6 +2914,8 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
         "classifier_model",
         "classifier_cost",
         "escalated",
+        "context_escalated",
+        "context_escalation_original_tier",
         "tier_boundaries",
         "reasoning_override_min_score",
         "conversation_continuing",

litellm/utils.py

@@ -8830,6 +8830,12 @@ class ProviderConfigManager:
             )
 
             return AzurePassthroughConfig()
+        elif LlmProviders.GIGACHAT == provider:
+            from litellm.llms.gigachat.passthrough.transformation import (
+                GigaChatPassthroughConfig,
+            )
+
+            return GigaChatPassthroughConfig()
         elif LlmProviders.WATSONX == provider:
             from litellm.llms.watsonx.passthrough.transformation import (
                 WatsonxPassthroughConfig,

model_prices_and_context_window.json

@@ -553,6 +553,26 @@
         "supports_response_schema": true,
         "supports_vision": true
     },
+    "amazon.nova-sonic-v1:0": {
+        "input_cost_per_audio_token": 3.4e-06,
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "bedrock",
+        "mode": "realtime",
+        "output_cost_per_audio_token": 1.36e-05,
+        "output_cost_per_token": 2.4e-07,
+        "supports_audio_input": true,
+        "supports_audio_output": true
+    },
+    "amazon.nova-2-sonic-v1:0": {
+        "input_cost_per_audio_token": 3e-06,
+        "input_cost_per_token": 3.3e-07,
+        "litellm_provider": "bedrock",
+        "mode": "realtime",
+        "output_cost_per_audio_token": 1.2e-05,
+        "output_cost_per_token": 2.75e-06,
+        "supports_audio_input": true,
+        "supports_audio_output": true
+    },
     "amazon.rerank-v1:0": {
         "input_cost_per_query": 0.001,
         "input_cost_per_token": 0.0,
@@ -19566,6 +19586,61 @@
         "supports_system_messages": true,
         "supports_tool_choice": true
     },
+    "friendliai/zai-org/GLM-5.3-Flash": {
+        "litellm_provider": "friendliai",
+        "supports_reasoning": true,
+        "supports_function_calling": true,
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 5e-07,
+        "cache_read_input_token_cost": 3e-08,
+        "supports_prompt_caching": true,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_native_structured_output": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "mode": "chat",
+        "comment": "Native multimodal GLM model for efficient coding and long-horizon agent tasks",
+        "source": "https://api.friendli.ai/serverless/v1/models",
+        "supports_vision": true,
+        "supports_image_input": true,
+        "supports_video_input": true
+    },
+    "friendliai/zai-org/GLM-5.3": {
+        "litellm_provider": "friendliai",
+        "supports_reasoning": true,
+        "supports_function_calling": true,
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "input_cost_per_token": 1.26e-06,
+        "output_cost_per_token": 3.96e-06,
+        "cache_read_input_token_cost": 2.34e-07,
+        "supports_prompt_caching": true,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_native_structured_output": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "mode": "chat",
+        "comment": "Flagship GLM model for long-horizon coding, agents, and complex project delivery",
+        "source": "https://api.friendli.ai/serverless/v1/models",
+        "supports_vision": false,
+        "supports_image_input": false
+    },
     "ft:babbage-002": {
         "deprecation_date": "2026-10-23",
         "input_cost_per_token": 1.6e-06,
@@ -24344,7 +24419,7 @@
         "supports_response_schema": true,
         "supports_vision": true
     },
-    "gigachat/GigaChat-2-Lite": {
+    "gigachat/GigaChat-2": {
         "input_cost_per_token": 0.0,
         "litellm_provider": "gigachat",
         "max_input_tokens": 128000,
@@ -24406,6 +24481,15 @@
         "output_cost_per_token": 0.0,
         "output_vector_size": 2560
     },
+    "gigachat/GigaEmbeddings-3B-2025-09": {
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "gigachat",
+        "max_input_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "embedding",
+        "output_cost_per_token": 0.0,
+        "output_vector_size": 2048
+    },
     "gmi/anthropic/claude-opus-4.5": {
         "input_cost_per_token": 5e-06,
         "litellm_provider": "gmi",

osv-scanner.toml

@@ -2,3 +2,8 @@
 id = "GHSA-w8v5-vhqr-4h9v"
 ignoreUntil = 2026-09-09
 reason = "diskcache has no fixed release published; remove this entry once one exists"
+
+[[IgnoredVulns]]
+id = "GHSA-h7x2-h6g9-p789"
+ignoreUntil = 2026-09-14
+reason = "mlflow has no fixed release published; remove this entry once one exists"

pyproject.toml

@@ -91,6 +91,11 @@ cli = [
 ]
 extra_proxy = [
     "prisma>=0.11.0,<1.0",
+    # Used by ProxyExtrasDBManager.spend_logs_is_partitioned() to detect a
+    # partitioned LiteLLM_SpendLogs and keep schema reconciliation from
+    # fighting its composite primary key.
+    "psycopg>=3.2,<4.0",
+    "psycopg-binary>=3.2,<4.0",
     "azure-identity>=1.25.2,<2.0",
     "azure-keyvault-secrets>=4.10.0,<5.0",
     # Not in PyPI proxy extra.

schema.prisma

@@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession {
 model LiteLLM_ShadowEvalJob {
   id                String   @id @default(cuid())
   group_id          String   // legs of one job share this; the API's job id
-  api_key_id        String   // hashed virtual key whose traffic this leg shadows
+  target_type       String   @default("key") // key | team | user
+  target_id         String   // hashed virtual key, team_id, or user_id whose traffic this leg shadows
   router_name       String   // the auto-router under evaluation, in either direction
   direction         String   @default("forward") // forward | reverse
   baseline_model    String?  // reverse only: the fixed model the router is judged against
   judge_model       String
   shadow_percentage Float
   max_turns         Int      // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
-  max_budget        Float?   // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
+  max_budget        Float?   // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
   created_at        DateTime @default(now())
   created_by        String?
   ends_at           DateTime
@@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob {
   stopped_by        String?  // operator who stopped it early; null when it ended on its own
 
   @@index([group_id])
-  @@index([api_key_id])
+  @@index([target_type, target_id])
   @@index([created_at])
 }
 

tests/documentation_tests/test_env_keys.py

@@ -33,6 +33,13 @@ EXCLUDED_ROLLOUT_FLAGS = {
     "LITELLM_RUST",
 }
 
+# Internal infrastructure tuning parameters for streaming/queue management
+# These are advanced settings with sensible defaults that most users should not modify
+EXCLUDED_INTERNAL_TUNING_VARS = {
+    "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS",
+    "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE",
+}
+
 EXCLUDED_TERMINAL_VARS = {
     "TERM",
     "TERM_PROGRAM",
@@ -50,7 +57,9 @@ EXCLUDED_TERMINAL_VARS = {
     "ALACRITTY_SOCKET",
 }
 
-EXCLUDED_KEYS = frozenset(EXCLUDED_TERMINAL_VARS | EXCLUDED_GUARD_ONLY_VARS | EXCLUDED_ROLLOUT_FLAGS)
+EXCLUDED_KEYS = frozenset(
+    EXCLUDED_TERMINAL_VARS | EXCLUDED_GUARD_ONLY_VARS | EXCLUDED_ROLLOUT_FLAGS | EXCLUDED_INTERNAL_TUNING_VARS
+)
 
 # Directories to skip (dependencies, venvs, caches) - only scan litellm source
 SKIP_DIRS = {

tests/e2e/management/test_model_tag_accessgroup_e2e.py

@@ -180,6 +180,12 @@ class ModelBlockBody(BaseModel):
     model_id: str
 
 
+class ModelBlockResponse(BaseModel):
+    model_config = ConfigDict(protected_namespaces=())
+    model_id: str
+    blocked: bool
+
+
 class ModelInfoBlockDetail(BaseModel):
     id: str | None = None
     blocked: bool | None = None
@@ -245,11 +251,6 @@ class TestModelRoutes:
     def test_block_then_unblock_persists_to_model_info(
         self, client: ManagementClient, resources: ResourceManager
     ) -> None:
-        """The blocked flag's persistence is read back from /model/info, not from the
-        /model/block response: that route currently returns a non-2xx serialization
-        envelope even though the DB write lands, so the /model/info read-back is the
-        authoritative persistence contract and keeps this test valid once the
-        response shape is fixed."""
         model_name = f"e2e-mgmt-model-block-{unique_marker()}"
         model_id = _create_db_model(client, resources, model_name)
 
@@ -257,27 +258,25 @@ class TestModelRoutes:
             f"{model_name!r} already reports blocked in /model/info before /model/block ran"
         )
 
-        _ = client.proxy.transport.send(
-            "/model/block",
-            headers=client.proxy.transport.master,
-            json=ModelBlockBody(model_id=model_id),
-        )
-        _ = _poll(
-            client.proxy,
-            lambda: True if _model_blocked_flag(client, model_id) is True else None,
-            f"/model/info never reported {model_name!r} blocked after /model/block",
-        )
-
-        _ = client.proxy.transport.send(
-            "/model/unblock",
-            headers=client.proxy.transport.master,
-            json=ModelBlockBody(model_id=model_id),
-        )
-        _ = _poll(
-            client.proxy,
-            lambda: True if _model_blocked_flag(client, model_id) is not True else None,
-            f"/model/info never cleared blocked for {model_name!r} after /model/unblock",
-        )
+        for action, expected in (("block", True), ("unblock", False)):
+            response = unwrap(
+                client.proxy.transport.post(
+                    f"/model/{action}",
+                    headers=client.proxy.transport.master,
+                    json=ModelBlockBody(model_id=model_id),
+                    response_type=ModelBlockResponse,
+                )
+            )
+            assert response.model_id == model_id
+            assert response.blocked is expected
+            _ = _poll(
+                client.proxy,
+                lambda want=expected: True
+                if _model_blocked_flag(client, model_id) is want
+                else None,
+                f"/model/info never reported blocked={expected} for {model_name!r} "
+                f"after /model/{action}",
+            )
 
 
 class TestTagRoutes:

tests/e2e/ui/tests/internal-user/internalUser.spec.ts

@@ -25,8 +25,7 @@ test.describe("Internal User", () => {
     const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox");
     await teamSelect.click();
     await page.keyboard.type(E2E_TEAM_CRUD_ALIAS);
-    const dropdown = page.locator('[data-slot="combobox-content"]:visible');
-    await expect(dropdown.getByText(E2E_TEAM_CRUD_ALIAS).first()).toBeVisible({ timeout: 5_000 });
+    await expect(page.getByRole("option", { name: E2E_TEAM_CRUD_ALIAS }).first()).toBeVisible({ timeout: 5_000 });
   });
 
   test("Team info page omits the Settings tab for non-admin members", async ({ page }) => {
@@ -98,12 +97,12 @@ test.describe("Internal User", () => {
 
     // Anchor on the user's own seeded key so the absence check below cannot
     // pass vacuously against an empty table.
-    await expect(page.locator("table tbody").getByText(E2E_INTERNAL_USER_KEY_ALIAS).first()).toBeVisible({
+    await expect(page.getByRole("row").filter({ hasText: E2E_INTERNAL_USER_KEY_ALIAS }).first()).toBeVisible({
       timeout: 10_000,
     });
 
     // The litellm-dashboard team is the proxy's internal bookkeeping team —
     // its keys must never leak into an internal user's Virtual Keys table.
-    await expect(page.locator("table tbody").getByText("litellm-dashboard")).toHaveCount(0);
+    await expect(page.getByRole("row").filter({ hasText: "litellm-dashboard" })).toHaveCount(0);
   });
 });

tests/e2e/ui/tests/internal-user/internalUserNoTeam.spec.ts

@@ -30,16 +30,13 @@ test.describe("Internal User with no team memberships", () => {
     const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox");
     await teamSelect.click();
 
-    const dropdown = page.locator('[data-slot="combobox-content"]:visible').first();
-    await expect(dropdown).toBeVisible({ timeout: 5_000 });
-
     // Wait for the settled-empty state, not a transient one. The dropdown shows
     // "Loading teams…" while teams load and only swaps in "No teams found" once
     // the request resolves with nothing (team_dropdown.tsx passes both copies to
     // PaginatedSearchSelect). Asserting on it means a regression where teams DO
     // load for this user fails here instead of racing a one-shot count() against
     // an in-flight request.
-    await expect(dropdown.getByText("No teams found")).toBeVisible({ timeout: 10_000 });
-    await expect(dropdown.getByRole("option")).toHaveCount(0);
+    await expect(page.getByText("No teams found")).toBeVisible({ timeout: 10_000 });
+    await expect(page.getByRole("option")).toHaveCount(0);
   });
 });

tests/e2e/ui/tests/internal-user/internalUserWithTeams.spec.ts

@@ -26,14 +26,11 @@ test.describe("Internal User with team memberships", () => {
     const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox");
     await teamSelect.click();
 
-    const dropdown = page.locator('[data-slot="combobox-content"]:visible').first();
-    await expect(dropdown).toBeVisible({ timeout: 5_000 });
-
     // All seeded memberships render, and nothing else does — proving the
     // dropdown is scoped to the user's teams rather than empty or unfiltered.
-    await expect(dropdown.getByText(E2E_TEAM_CRUD_ALIAS, { exact: true })).toBeVisible({ timeout: 10_000 });
-    await expect(dropdown.getByText(E2E_TEAM_ORG_ALIAS, { exact: true })).toBeVisible();
-    await expect(dropdown.getByText(E2E_TEAM_KEYGEN_ALIAS, { exact: true })).toBeVisible();
-    await expect(dropdown.getByRole("option")).toHaveCount(3);
+    await expect(page.getByRole("option", { name: E2E_TEAM_CRUD_ALIAS })).toBeVisible({ timeout: 10_000 });
+    await expect(page.getByRole("option", { name: E2E_TEAM_ORG_ALIAS })).toBeVisible();
+    await expect(page.getByRole("option", { name: E2E_TEAM_KEYGEN_ALIAS })).toBeVisible();
+    await expect(page.getByRole("option")).toHaveCount(3);
   });
 });

tests/e2e/ui/tests/internal-viewer/internalViewer.spec.ts

@@ -59,9 +59,9 @@ test.describe("Internal Viewer", () => {
     await expect(page.getByRole("button", { name: /Create New Key/i })).toHaveCount(0);
 
     // Open the viewer's own key info page
-    const keyRow = page.locator("tr", { hasText: E2E_VIEWER_KEY_ALIAS });
+    const keyRow = page.getByRole("row").filter({ hasText: E2E_VIEWER_KEY_ALIAS });
     await expect(keyRow).toBeVisible({ timeout: 10_000 });
-    await keyRow.locator("button").first().click();
+    await keyRow.getByRole("button", { name: E2E_VIEWER_KEY_ALIAS }).click();
     await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 });
 
     // None of the destructive / mutating actions should render

tests/e2e/ui/tests/logs/logs.spec.ts

@@ -18,12 +18,11 @@ import { openPlayground, selectModel, sendMessage } from "../../helpers/playgrou
 
 const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
 
-/**
- * Walking up from the label is the only stable handle: the header carries no role, test id or class,
- * and its copy button is icon-only with a hover-only tooltip.
- */
-const sectionHeader = (drawer: Locator, label: "Input" | "Output"): Locator =>
-  drawer.getByText(label, { exact: true }).locator("xpath=../../..");
+const sectionToggle = (drawer: Locator, label: "Input" | "Output"): Locator =>
+  drawer.getByRole("button", { name: new RegExp(`^${label}\\b`) });
+
+const sectionCopy = (drawer: Locator, label: "Input" | "Output"): Locator =>
+  drawer.getByRole("button", { name: `Copy ${label.toLowerCase()}` });
 
 /** Every tab stays mounted, so the DOM holds four tables at once; scope to the visible one. */
 const requestLogsRows = (page: PlaywrightPage): Locator =>
@@ -119,14 +118,14 @@ test.describe("Logs page", () => {
     await expect(drawer).toBeVisible({ timeout: 20_000 });
 
     // Copy request: the Input card's copy button puts the prompt on the clipboard.
-    await sectionHeader(drawer, "Input").getByRole("button").click();
+    await sectionCopy(drawer, "Input").click();
     await expect(page.getByText("Input copied")).toBeVisible({
       timeout: 10_000,
     });
     expect(await page.evaluate(() => navigator.clipboard.readText())).toContain(prompt);
 
     // Copy response: the Output card's copy button puts the completion on it.
-    await sectionHeader(drawer, "Output").getByRole("button").click();
+    await sectionCopy(drawer, "Output").click();
     await expect(page.getByText("Output copied")).toBeVisible({
       timeout: 10_000,
     });
@@ -149,24 +148,15 @@ test.describe("Logs page", () => {
       timeout: 20_000,
     });
 
-    // The body collapses via `max-height: 0; overflow: hidden`, which zeroes its own bounding
-    // box, so the wrapper reads as hidden while the clipped text node inside it does not.
-    const header = sectionHeader(drawer, "Input");
-    const body = header.locator("xpath=following-sibling::div[1]");
-    await expect(header.locator(".lucide-chevron-up")).toBeVisible();
-    await expect(body).toBeVisible();
+    const toggle = sectionToggle(drawer, "Input");
+    await expect(toggle).toHaveAttribute("aria-expanded", "true");
+    await expect(drawer.getByText(prompt, { exact: false })).toBeVisible();
 
-    await header.click();
-    await expect(header.locator(".lucide-chevron-down")).toBeVisible({
-      timeout: 10_000,
-    });
-    await expect(body).toBeHidden({ timeout: 10_000 });
+    await toggle.click();
+    await expect(toggle).toHaveAttribute("aria-expanded", "false", { timeout: 10_000 });
 
-    await header.click();
-    await expect(header.locator(".lucide-chevron-up")).toBeVisible({
-      timeout: 10_000,
-    });
-    await expect(body).toBeVisible({ timeout: 10_000 });
+    await toggle.click();
+    await expect(toggle).toHaveAttribute("aria-expanded", "true", { timeout: 10_000 });
     await expect(drawer.getByText(prompt, { exact: false })).toBeVisible({
       timeout: 10_000,
     });

tests/e2e/ui/tests/mcp/mcpServerEdit.spec.ts

@@ -73,7 +73,7 @@ test.describe("MCP Servers - edit and delete", () => {
   test("Deleting a server removes it", async ({ page }) => {
     expect(await findServerByName(page, serverName), `created server ${serverName} exists`).toBeTruthy();
 
-    const card = page.getByTestId("mcp-servers-grid").locator("div").filter({ hasText: serverName }).first();
+    const card = page.getByTestId("mcp-servers-grid").getByRole("button", { name: serverName });
     await card.getByRole("button", { name: "Server actions" }).click();
     await page.getByRole("menuitem", { name: "Delete" }).click();
 

tests/e2e/ui/tests/migration/migratedPages.spec.ts

@@ -35,17 +35,12 @@ async function expectRendered(page: Page) {
  */
 async function clickSidebar(page: Page, segment: string) {
   const link = sidebar(page).locator(`a[href$="/ui/${segment}"]`).first();
+  const collapsedGroups = sidebar(page).getByRole("button", { expanded: false });
   for (let i = 0; i < 8 && !(await link.isVisible().catch(() => false)); i++) {
-    // A collapsed group is a menu item with a group-toggle button but no
-    // rendered submenu yet; clicking the toggle expands it.
-    const collapsedGroup = sidebar(page)
-      .locator(
-        '[data-slot="sidebar-menu-item"]:has(> [data-slot="sidebar-menu-button"]):not(:has(> [data-slot="sidebar-menu-sub"])) > [data-slot="sidebar-menu-button"]',
-      )
-      .first();
-    if (!(await collapsedGroup.isVisible().catch(() => false))) break;
-    await collapsedGroup.click();
-    await page.waitForTimeout(250);
+    const stillCollapsed = await collapsedGroups.count();
+    if (stillCollapsed === 0) break;
+    await collapsedGroups.first().click();
+    await expect(collapsedGroups).toHaveCount(stillCollapsed - 1);
   }
   await link.click();
 }

tests/e2e/ui/tests/modelsPage/addModel.spec.ts

@@ -188,7 +188,7 @@ test.describe("Add Model", () => {
     await expect(resultsModal).toBeHidden({ timeout: 5_000 });
 
     const created = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => {
-      await page.getByRole("button", { name: "Add Model" }).last().click();
+      await page.getByTestId("add-model-btn").click();
     });
     expect(created.model_name, "the model is created under the name that was typed").toBe(publicName);
     expect(created.litellm_params?.api_base, "the api base survives the form").toBe(MOCK_LLM_BASE);
@@ -335,7 +335,7 @@ test.describe("Add Model", () => {
 
     // Click Add Model button by its text
     const created = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => {
-      await page.getByRole("button", { name: "Add Model" }).last().click();
+      await page.getByTestId("add-model-btn").click();
     });
     // The form sends custom_llm_provider separately from the name, so both halves have to arrive.
     expect(created.model_name, "the selected model is what goes on the wire").toBe("claude-haiku-4-5");
@@ -348,11 +348,9 @@ test.describe("Add Model", () => {
     // Navigate to All Models tab
     await page.getByRole("tab", { name: "All Models" }).click();
     await page.waitForLoadState("networkidle");
-    await page.waitForTimeout(2000);
 
     // Search for the model we just added
     await page.getByPlaceholder("Search model names").fill("claude-haiku-4-5");
-    await page.waitForTimeout(1000);
 
     // Verify the model appears in the results count (not "Showing 0 results")
     await expect(page.getByTestId("pagination-range")).toHaveText(/Showing \d+-\d+ of \d+/, {
@@ -360,8 +358,9 @@ test.describe("Add Model", () => {
     });
 
     // Verify the model name appears in the table body
-    const tableBody = page.locator("table tbody");
-    await expect(tableBody.getByText("claude-haiku-4-5").first()).toBeVisible({ timeout: 15_000 });
+    await expect(page.getByRole("row").filter({ hasText: "claude-haiku-4-5" })).not.toHaveCount(0, {
+      timeout: 15_000,
+    });
 
     // A row proves the name is there, not what the deployment routes to.
     const stored = await findDeploymentByName(page, "claude-haiku-4-5");
@@ -414,11 +413,11 @@ test.describe("Add Model", () => {
       const teamDropdown = page.getByTestId("team-dropdown").getByRole("combobox");
       await expect(teamDropdown).toBeVisible({ timeout: 5_000 });
       await teamDropdown.click();
-      const teamOption = page.locator('[data-slot="combobox-content"]:visible').getByText(E2E_TEAM_CRUD_ID).first();
+      const teamOption = page.getByRole("option", { name: E2E_TEAM_CRUD_ID }).first();
       await expect(teamOption).toBeVisible({ timeout: 5_000 });
       await teamOption.click();
 
-      await page.getByRole("button", { name: "Add Model" }).last().click();
+      await page.getByTestId("add-model-btn").click();
 
       // Scope to the toast container so a stale toast can't satisfy this.
       await expect(page.locator("[data-sonner-toast]").getByText("created successfully").last()).toBeVisible({
@@ -428,12 +427,9 @@ test.describe("Add Model", () => {
       // The Models table renders team-scoped models with the team id in the row.
       await page.getByRole("tab", { name: "All Models" }).click();
       await page.waitForLoadState("networkidle");
-      // networkidle fires before the table finishes re-rendering.
-      await page.waitForTimeout(2000);
 
       await page.getByPlaceholder("Search model names").fill("cohere");
-      await page.waitForTimeout(1000);
-
+  
       // Clearer failure than timing out on a row assertion when the table is empty.
       await expect(page.getByTestId("pagination-range")).toHaveText(/Showing \d+-\d+ of \d+/, {
         timeout: 15_000,
@@ -442,7 +438,7 @@ test.describe("Add Model", () => {
       // Pin to one row carrying both the name and the team, so the sibling test's
       // team-less cohere row can't satisfy it.
       const teamCohereRow = page
-        .locator("table tbody tr")
+        .getByRole("row")
         .filter({ hasText: "cohere/" })
         .filter({ hasText: E2E_TEAM_CRUD_ID });
       await expect(teamCohereRow).toHaveCount(1, { timeout: 15_000 });
@@ -468,7 +464,7 @@ test.describe("Add Model", () => {
 
     // Click Add Model button by its text
     const created = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => {
-      await page.getByRole("button", { name: "Add Model" }).last().click();
+      await page.getByTestId("add-model-btn").click();
     });
     // A wildcard with the star stripped becomes a plain "cohere" deployment that matches nothing.
     expect(created.model_name, "the wildcard route goes on the wire intact").toBe("cohere/*");
@@ -479,11 +475,9 @@ test.describe("Add Model", () => {
     // Navigate to All Models tab
     await page.getByRole("tab", { name: "All Models" }).click();
     await page.waitForLoadState("networkidle");
-    await page.waitForTimeout(2000);
 
     // Search for the wildcard model
     await page.getByPlaceholder("Search model names").fill("cohere");
-    await page.waitForTimeout(1000);
 
     // Verify the model appears in the results count (not "Showing 0 results")
     await expect(page.getByTestId("pagination-range")).toHaveText(/Showing \d+-\d+ of \d+/, {
@@ -491,8 +485,7 @@ test.describe("Add Model", () => {
     });
 
     // Verify the wildcard model appears in the table body (wildcard models show as "cohere/*")
-    const tableBody = page.locator("table tbody");
-    await expect(tableBody.getByText("cohere/").first()).toBeVisible({ timeout: 15_000 });
+    await expect(page.getByRole("row").filter({ hasText: "cohere/" })).not.toHaveCount(0, { timeout: 15_000 });
 
     // "cohere/" in the table also matches a plain cohere deployment; require the wildcard exactly.
     const stored = await findDeploymentByName(page, "cohere/*");

tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts

@@ -17,49 +17,54 @@ async function openTemplateSelect(page: PlaywrightPage) {
   return trigger;
 }
 
-function pollPixelsBelowTrigger(trigger: Locator, popup: Locator) {
+async function boxes(trigger: Locator, options: Locator) {
+  const triggerBox = await trigger.boundingBox();
+  const optionsBox = await options.boundingBox();
+  return triggerBox && optionsBox ? { triggerBox, optionsBox } : null;
+}
+
+const clippedPopup = (page: PlaywrightPage) => page.locator('[data-slot="select-content"]');
+
+function pollOptionsOpenBelowTrigger(trigger: Locator, options: Locator) {
   return expect.poll(async () => {
-    const triggerBox = await trigger.boundingBox();
-    const popupBox = await popup.boundingBox();
-    if (!triggerBox || !popupBox) return null;
-    return popupBox.y - (triggerBox.y + triggerBox.height);
+    const box = await boxes(trigger, options);
+    return box && box.optionsBox.y >= box.triggerBox.y + box.triggerBox.height;
   });
 }
 
-function pollPopupOverlapsTrigger(trigger: Locator, popup: Locator) {
+function pollOptionsCoverTrigger(trigger: Locator, options: Locator) {
   return expect.poll(async () => {
-    const triggerBox = await trigger.boundingBox();
-    const popupBox = await popup.boundingBox();
-    if (!triggerBox || !popupBox) return null;
-    return popupBox.y < triggerBox.y + triggerBox.height && popupBox.y + popupBox.height > triggerBox.y;
+    const box = await boxes(trigger, options);
+    return (
+      box &&
+      box.optionsBox.y < box.triggerBox.y + box.triggerBox.height &&
+      box.optionsBox.y + box.optionsBox.height > box.triggerBox.y
+    );
   });
 }
 
 test.describe("Auto Router template select anchoring", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
-  test("opens the options below the trigger rather than over it", async ({ page }) => {
+  test("opens the options below the trigger when there is room below it", async ({ page }) => {
     await page.setViewportSize({ width: 1280, height: 900 });
     const trigger = await openTemplateSelect(page);
+    await trigger.scrollIntoViewIfNeeded();
 
     await trigger.click();
-    const popup = page.locator('[data-slot="select-content"]');
-    await expect(popup).toBeVisible();
+    await expect(page.getByRole("listbox")).toBeVisible();
 
-    // Item-aligned mode reports "none" and puts the active item over the trigger.
-    await expect(popup).toHaveAttribute("data-side", "bottom");
-    await pollPixelsBelowTrigger(trigger, popup).toBeGreaterThanOrEqual(0);
+    await pollOptionsOpenBelowTrigger(trigger, clippedPopup(page)).toBe(true);
   });
 
-  test("flips above the trigger instead of covering it when there is no room below", async ({ page }) => {
+  test("keeps the trigger uncovered when the options open with no room below it", async ({ page }) => {
     await page.setViewportSize({ width: 1280, height: 560 });
     const trigger = await openTemplateSelect(page);
     await trigger.scrollIntoViewIfNeeded();
 
     await trigger.click();
-    const popup = page.locator('[data-slot="select-content"]');
-    await expect(popup).toBeVisible();
+    await expect(page.getByRole("listbox")).toBeVisible();
 
-    await pollPopupOverlapsTrigger(trigger, popup).toBe(false);
+    await pollOptionsCoverTrigger(trigger, clippedPopup(page)).toBe(false);
   });
 });

tests/e2e/ui/tests/modelsPage/responsiveHeader.spec.ts

@@ -7,9 +7,7 @@ test.describe("Models and Endpoints responsive header", () => {
     viewport: { width: 900, height: 720 },
   });
 
-  test("keeps the refresh action on the same row as the tabs", async ({
-    page,
-  }) => {
+  test("keeps the refresh action on the same row as the tabs", async ({ page }) => {
     await page.goto("/ui");
     await page
       .getByRole("complementary")
@@ -26,8 +24,8 @@ test.describe("Models and Endpoints responsive header", () => {
     expect(tabsBox).not.toBeNull();
     expect(refreshBox).not.toBeNull();
 
-    const tabsCenterY = tabsBox!.y + tabsBox!.height / 2;
     const refreshCenterY = refreshBox!.y + refreshBox!.height / 2;
-    expect(Math.abs(tabsCenterY - refreshCenterY)).toBeLessThanOrEqual(2);
+    const sharesARow = refreshCenterY > tabsBox!.y && refreshCenterY < tabsBox!.y + tabsBox!.height;
+    expect(sharesARow, "refresh wrapped onto its own row below the tabs").toBe(true);
   });
 });

tests/e2e/ui/tests/proxy-admin/keys.spec.ts

@@ -43,7 +43,7 @@ test.describe("Proxy Admin - Keys", () => {
     const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox");
     await teamSelect.click();
     await page.keyboard.type(E2E_TEAM_CRUD_ALIAS);
-    await page.locator('[data-slot="combobox-content"]:visible').getByText(E2E_TEAM_CRUD_ALIAS).first().click();
+    await page.getByRole("option", { name: E2E_TEAM_CRUD_ALIAS }).first().click();
 
     // Select models — the popup is portaled to the body, so scope options to the page.
     await page.getByRole("combobox", { name: "Select models" }).click();
@@ -74,10 +74,9 @@ test.describe("Proxy Admin - Keys", () => {
     const before = await findKeyByAlias(page, E2E_REGENERATE_KEY_ALIAS);
     expect(before?.token, `seeded key ${E2E_REGENERATE_KEY_ALIAS} has a token`).toBeTruthy();
 
-    // Key IDs are rendered as buttons in the table
-    const keyRow = page.locator("tr", { hasText: E2E_REGENERATE_KEY_ALIAS });
+    const keyRow = page.getByRole("row").filter({ hasText: E2E_REGENERATE_KEY_ALIAS });
     await expect(keyRow).toBeVisible({ timeout: 10_000 });
-    await keyRow.locator("button").first().click();
+    await keyRow.getByRole("button", { name: E2E_REGENERATE_KEY_ALIAS }).click();
 
     await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 });
 
@@ -109,9 +108,9 @@ test.describe("Proxy Admin - Keys", () => {
     const before = await findKeyByAlias(page, E2E_UPDATE_LIMITS_KEY_ALIAS);
     expect(before, `seeded key ${E2E_UPDATE_LIMITS_KEY_ALIAS} exists`).toBeTruthy();
 
-    const keyRow = page.locator("tr", { hasText: E2E_UPDATE_LIMITS_KEY_ALIAS });
+    const keyRow = page.getByRole("row").filter({ hasText: E2E_UPDATE_LIMITS_KEY_ALIAS });
     await expect(keyRow).toBeVisible({ timeout: 10_000 });
-    await keyRow.locator("button").first().click();
+    await keyRow.getByRole("button", { name: E2E_UPDATE_LIMITS_KEY_ALIAS }).click();
 
     await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 });
 
@@ -147,9 +146,9 @@ test.describe("Proxy Admin - Keys", () => {
     await navigateToPage(page, Page.ApiKeys);
     await dismissFeedbackPopup(page);
 
-    const keyRow = page.locator("tr", { hasText: E2E_DELETE_KEY_ALIAS });
+    const keyRow = page.getByRole("row").filter({ hasText: E2E_DELETE_KEY_ALIAS });
     await expect(keyRow).toBeVisible({ timeout: 10_000 });
-    await keyRow.locator("button").first().click();
+    await keyRow.getByRole("button", { name: E2E_DELETE_KEY_ALIAS }).click();
 
     await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 });
 

tests/e2e/ui/tests/settings/scim.spec.ts

@@ -0,0 +1,53 @@
+import { test, expect, Page as PlaywrightPage } from "@playwright/test";
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { Page } from "../../fixtures/pages";
+import { navigateToPage } from "../../helpers/navigation";
+
+const rootPath = (): string => process.env.SERVER_ROOT_PATH ?? "";
+
+async function createScimTokenViaUi(page: PlaywrightPage, alias: string): Promise<string> {
+  await navigateToPage(page, Page.AdminPanel);
+  await page.getByRole("tab", { name: "SCIM" }).click();
+
+  await expect(page.getByText("SCIM Tenant URL")).toBeVisible();
+  await expect(page.locator("input[disabled]").first()).toHaveValue(/\/scim\/v2$/);
+
+  await page.getByLabel("Token Name").fill(alias);
+  await page.getByRole("button", { name: "Create SCIM Token" }).click();
+
+  await expect(page.getByText(/copy this token now/i)).toBeVisible({ timeout: 15_000 });
+  const token = await page.locator('input[type="password"]').inputValue();
+  expect(token, "the one-time token panel shows a usable virtual key").toMatch(/^sk-/);
+  return token;
+}
+
+test.describe("Admin Settings - SCIM", () => {
+  test.use({ storageState: ADMIN_STORAGE_PATH });
+
+  test("Create SCIM Token shows the token once and offers to create another", async ({ page }) => {
+    await createScimTokenViaUi(page, `e2e-scim-ui-${Date.now()}`);
+
+    await page.getByRole("button", { name: "Create Another Token" }).click();
+    await expect(page.getByRole("button", { name: "Create SCIM Token" })).toBeVisible();
+    await expect(page.getByText(/copy this token now/i)).toBeHidden();
+  });
+
+  test("a UI-minted SCIM token authorizes the SCIM API", async ({ page, request }) => {
+    test.skip(!process.env.LITELLM_LICENSE, "LITELLM_LICENSE not set in test env — /scim/v2 is premium-gated");
+
+    const token = await createScimTokenViaUi(page, `e2e-scim-api-${Date.now()}`);
+
+    const denied = await request.get(`${rootPath()}/scim/v2/Groups`, {
+      headers: { Authorization: "Bearer sk-not-a-real-key" },
+    });
+    expect(denied.status(), "an unknown key must not reach SCIM").toBe(401);
+
+    const res = await request.get(`${rootPath()}/scim/v2/Groups`, {
+      headers: { Authorization: `Bearer ${token}` },
+    });
+    expect(res.status(), `SCIM Groups listing failed: ${await res.text()}`).toBe(200);
+    const body = await res.json();
+    expect(body.schemas, "SCIM answers with a ListResponse").toContain("urn:ietf:params:scim:api:messages:2.0:ListResponse");
+    expect(Array.isArray(body.Resources), "SCIM ListResponse carries a Resources array").toBe(true);
+  });
+});

tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts

@@ -230,7 +230,7 @@ test.describe("Team Admin", () => {
     const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox");
     await teamSelect.click();
     await page.keyboard.type(E2E_TEAM_CRUD_ALIAS);
-    await page.locator('[data-slot="combobox-content"]:visible').getByText(E2E_TEAM_CRUD_ALIAS).first().click();
+    await page.getByRole("option", { name: E2E_TEAM_CRUD_ALIAS }).first().click();
 
     // Models — pick "All Team Models". The popup is portaled to the body, so
     // scope the option lookup to the page.

tests/e2e/ui/tests/usage/usagePage.spec.ts

@@ -51,20 +51,19 @@ test.describe("Usage page", () => {
     const card = await openUsage(page);
 
     // Table view (the default): the key is listed by its alias.
-    const row = card.locator("tbody tr").filter({ hasText: alias });
+    const row = card.getByRole("row").filter({ hasText: alias });
     await expect(row, `${alias} missing from Top Virtual Keys`).toHaveCount(1, {
       timeout: 30_000,
     });
 
     // Chart view swaps the table out for the bar chart, and back.
     await card.getByText("Chart View", { exact: true }).click();
-    await expect(card.locator("tbody tr")).toHaveCount(0, { timeout: 10_000 });
+    await expect(card.getByRole("table")).toHaveCount(0, { timeout: 10_000 });
     await card.getByText("Table View", { exact: true }).click();
     await expect(row).toHaveCount(1, { timeout: 10_000 });
 
-    // Clicking the Key ID cell fetches key info and opens the detail panel.
     // The alias is already in the row behind the modal, so match the panel's own controls.
-    await row.locator("td").first().click();
+    await row.getByRole("button", { name: token }).click();
     const keyInfo = page.getByRole("tab", { name: "Overview", exact: true });
     await expect(keyInfo, "key info panel did not open").toBeVisible({
       timeout: 20_000,

tests/e2e/ui/tests/users/searchUsers.spec.ts

@@ -1,91 +1,52 @@
-import { test, expect, Page } from "@playwright/test";
+import { test, expect, Page as PlaywrightPage } from "@playwright/test";
 import { ADMIN_STORAGE_PATH } from "../../constants";
-test.skip("Internal Users Search", () => {
+import { Page } from "../../fixtures/pages";
+import { navigateToPage } from "../../helpers/navigation";
+
+const userRows = (page: PlaywrightPage) => page.getByRole("row").filter({ has: page.getByRole("cell") });
+
+async function goToInternalUsers(page: PlaywrightPage) {
+  await navigateToPage(page, Page.Users);
+  await expect(page.getByRole("columnheader", { name: "User ID" })).toBeVisible({ timeout: 30_000 });
+  await expect(userRows(page)).not.toHaveCount(0, { timeout: 30_000 });
+}
+
+test.describe("Internal Users Search", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
-  async function goToInternalUsers(page: Page) {
-    await page.goto("/ui");
-
-    const tab = page.getByRole("menuitem", { name: "Internal User" });
-    await expect(tab).toBeVisible();
-    await tab.click();
-
-    await expect(page.locator("tbody tr").first()).toBeVisible();
-    await expect(page.locator('[data-slot="skeleton"]')).toHaveCount(0);
-  }
-
-  test("can search users by email", async ({ page }) => {
+  test("narrows the table to the matching email, and restores it when cleared", async ({ page }) => {
     await goToInternalUsers(page);
 
-    const rows = page.locator("tbody tr");
-    const searchInput = page.getByPlaceholder("Search by email...");
+    const search = page.getByPlaceholder("Search by email…");
+    await expect(search).toBeVisible();
 
-    await expect(searchInput).toBeVisible();
+    await search.fill("noteam@");
+    await expect(userRows(page)).toHaveCount(1, { timeout: 30_000 });
+    await expect(userRows(page).first()).toContainText("noteam@test.local");
 
-    // Ensure initial data is loaded
-    const initialCount = await rows.count();
-    expect(initialCount).toBeGreaterThan(0);
-
-    // 🔹 Apply filter + wait for backend response
-    await Promise.all([
-      page.waitForResponse(
-        (res) =>
-          res.url().includes("/user/list") &&
-          res.url().includes("user_email=test%40") && // encoded "test@"
-          res.status() === 200,
-      ),
-      searchInput.fill("test@"),
-    ]);
-    await page.waitForTimeout(5000);
-    const filteredCount = await rows.count();
-    await expect(filteredCount).toBeLessThan(initialCount);
-
-    // 🔹 Clear filter + wait for unfiltered request
-    await Promise.all([
-      page.waitForResponse(
-        (res) => res.url().includes("/user/list") && !res.url().includes("user_email=") && res.status() === 200,
-      ),
-      searchInput.clear(),
-    ]);
-
-    const resetCount = await rows.count();
-    await expect(resetCount).toBe(initialCount);
+    await search.clear();
+    await expect(userRows(page).filter({ hasText: "admin@test.local" })).not.toHaveCount(0, { timeout: 30_000 });
   });
 
-  test("can filter users by user ID and SSO ID", async ({ page }) => {
+  test("filters the table down to one user by user ID", async ({ page }) => {
     await goToInternalUsers(page);
-    const rows = page.locator("tbody tr");
 
-    // Ensure initial data is loaded
-    const initialCount = await rows.count();
-    expect(initialCount).toBeGreaterThan(0);
+    await page.getByRole("button", { name: "Filters" }).click();
+    await page.getByTestId("users-filter-user-id").fill("e2e-internal-noteam");
+    await page.getByTestId("filter-drawer-apply").click();
 
-    const filtersButton = page.getByRole("button", {
-      name: "Filters",
-      exact: true,
-    });
-    await filtersButton.click();
+    await expect(userRows(page)).toHaveCount(1, { timeout: 30_000 });
+    await expect(userRows(page).first()).toContainText("noteam@test.local");
+  });
 
-    const userIdInput = page.getByPlaceholder("Filter by User ID");
-    const ssoIdInput = page.getByPlaceholder("Filter by SSO ID");
-    await Promise.all([
-      page.waitForResponse(
-        (res) => res.url().includes("/user/list") && res.url().includes("user_ids=user") && res.status() === 200,
-      ),
-      userIdInput.fill("user"),
-    ]);
+  test("shows no users when the SSO ID matches nobody", async ({ page }) => {
+    await goToInternalUsers(page);
 
-    await Promise.all([
-      page.waitForResponse(
-        (res) =>
-          res.url().includes("/user/list") &&
-          res.url().includes("user_ids=user") &&
-          res.url().includes("sso_user_ids=sso") &&
-          res.status() === 200,
-      ),
-      ssoIdInput.fill("sso"),
-    ]);
-    const combinedFilteredCount = await rows.count();
-    await expect(combinedFilteredCount).toBeLessThan(initialCount);
+    await page.getByRole("button", { name: "Filters" }).click();
+    await page.getByTestId("users-filter-sso-id").fill("e2e-sso-id-that-matches-nobody");
+    await page.getByTestId("filter-drawer-apply").click();
+
+    await expect(page.getByText("No users found")).toBeVisible({ timeout: 30_000 });
+    await expect(userRows(page).filter({ hasText: "noteam@test.local" })).toHaveCount(0);
   });
 });

tests/e2e/ui/tests/users/viewInternalUsers.spec.ts

@@ -1,54 +1,29 @@
-import { test, expect, Page } from "@playwright/test";
+import { test, expect, Page as PlaywrightPage } from "@playwright/test";
 import { ADMIN_STORAGE_PATH } from "../../constants";
+import { Page } from "../../fixtures/pages";
+import { navigateToPage } from "../../helpers/navigation";
 
-test.skip("Internal Users Page", () => {
+async function goToInternalUsers(page: PlaywrightPage) {
+  await navigateToPage(page, Page.Users);
+  await expect(page.getByRole("columnheader", { name: "User ID" })).toBeVisible({ timeout: 30_000 });
+  await expect(userRows(page)).not.toHaveCount(0, { timeout: 30_000 });
+}
+
+const userRows = (page: PlaywrightPage) => page.getByRole("row").filter({ has: page.getByRole("cell") });
+
+test.describe("Internal Users Page", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
-  async function goToInternalUsers(page: Page) {
-    await page.goto("/ui");
-
-    const internalUserTab = page.getByRole("menuitem", { name: "Internal User" });
-    await expect(internalUserTab).toBeVisible();
-    await internalUserTab.click();
-
-    const firstRow = page.locator("tbody tr").first();
-    await expect(firstRow).toBeVisible();
-    await expect(page.locator('[data-slot="skeleton"]')).toHaveCount(0);
-  }
-
-  test("renders internal users table correctly", async ({ page }) => {
+  test("lists the seeded users under the identifying columns", async ({ page }) => {
     await goToInternalUsers(page);
 
-    const rows = page.locator("tbody tr");
-    const rowCount = await rows.count();
-    expect(rowCount).toBeGreaterThan(0);
-
-    const userIdHeader = page.getByRole("columnheader", { name: "User ID" });
-    await expect(userIdHeader).toBeVisible();
-
-    const virtualKeysHeader = page.getByRole("columnheader", { name: "Virtual Keys" });
-    await expect(virtualKeysHeader).toBeVisible();
+    await expect(page.getByRole("columnheader", { name: "User ID" })).toBeVisible();
+    await expect(page.getByRole("columnheader", { name: "Virtual Keys" })).toBeVisible();
   });
 
-  test("pagination controls work correctly", async ({ page }) => {
+  test("cannot page backwards off the first page", async ({ page }) => {
     await goToInternalUsers(page);
 
-    const paginationInfo = page.locator(".text-sm.text-gray-700");
-    const prevButton = page.getByRole("button", { name: "Previous" });
-    const nextButton = page.getByRole("button", { name: "Next" });
-
-    const infoText = (await paginationInfo.textContent()) || "";
-
-    // On first page, Previous should be disabled
-    if (infoText.includes("1 -")) {
-      await expect(prevButton).toBeDisabled();
-    }
-
-    await page.waitForTimeout(1000);
-    // Check if there are more pages
-    const hasMorePages = infoText.includes("of") && !infoText.endsWith("25 of 25");
-    if (hasMorePages) {
-      await expect(nextButton).toBeEnabled();
-    }
+    await expect(page.getByRole("button", { name: "Go to previous page" })).toBeDisabled();
   });
 });

tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py

@@ -681,3 +681,25 @@ class TestSpendLogsPartitionDetectionSchemaScope:
     def test_only_partitioned_relations_match(self, monkeypatch):
         query, _ = self._detect(monkeypatch, "postgresql://u:p@localhost:5432/db")
         assert "pg_partitioned_table" in query
+
+
+class TestSpendLogsPartitionDetectionMissingPsycopg:
+    """psycopg ships in the `extra_proxy` install, but a stripped-down image
+    can still lack it. When it does, detection must fail closed to False
+    (never crash the migration path) and say so loudly, because a silent
+    False here is what let a genuinely partitioned LiteLLM_SpendLogs hit the
+    unfiltered primary-key rewrite in production."""
+
+    def test_missing_psycopg_returns_false(self, monkeypatch):
+        monkeypatch.setitem(sys.modules, "psycopg", None)
+        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:5432/db")
+        assert ProxyExtrasDBManager.spend_logs_is_partitioned() is False
+
+    def test_missing_psycopg_logs_a_warning(self, monkeypatch, caplog):
+        monkeypatch.setitem(sys.modules, "psycopg", None)
+        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:5432/db")
+        with caplog.at_level("WARNING", logger="litellm_proxy_extras"):
+            ProxyExtrasDBManager.spend_logs_is_partitioned()
+        assert any(
+            "psycopg is not installed" in record.message for record in caplog.records
+        )

tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json

@@ -11,7 +11,7 @@
     "user": "",
     "team_id": "",
     "organization_id": "",
-    "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}",
+    "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"router_metadata\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}",
     "cache_key": "Cache OFF",
     "spend": 0.00022500000000000002,
     "total_tokens": 30,

tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py

@@ -0,0 +1,62 @@
+from typing import Final
+from unittest.mock import MagicMock
+
+import pytest
+
+import litellm
+from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
+from litellm.endpoints.speech.speech_to_completion_bridge.transformation import (
+    SpeechToCompletionBridgeTransformationHandler,
+)
+
+GEMINI_TTS_MODEL: Final = "gemini-3.1-flash-tts-preview"
+
+
+def _bridge_request(response_format: str | None) -> dict:
+    optional_params: Final = (
+        {"temperature": 0.4} if response_format is None else {"temperature": 0.4, "response_format": response_format}
+    )
+    return SpeechToCompletionBridgeTransformationHandler().transform_request(
+        model=GEMINI_TTS_MODEL,
+        input="Hello from LiteLLM",
+        voice="Kore",
+        optional_params=optional_params,
+        litellm_params={},
+        headers={},
+        litellm_logging_obj=MagicMock(),
+        custom_llm_provider="gemini",
+    )
+
+
+@pytest.mark.parametrize("response_format", ["wav", "mp3", "pcm", None])
+def test_gemini_tts_request_keeps_speech_response_format_out_of_chat_params(response_format: str | None) -> None:
+    request: Final = _bridge_request(response_format)
+
+    assert "response_format" not in request
+    assert request["audio"] == {"voice": "Kore", "format": "pcm16"}
+    assert request["temperature"] == 0.4
+    assert request["modalities"] == ["audio"]
+
+    gemini_params: Final = litellm.get_optional_params(
+        model=GEMINI_TTS_MODEL,
+        custom_llm_provider="gemini",
+        **{param: value for param, value in request.items() if param in OPENAI_CHAT_COMPLETION_PARAMS},
+    )
+    assert gemini_params["speechConfig"] == {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}}
+    assert "responseMimeType" not in gemini_params
+
+
+def test_non_gemini_request_forwards_speech_response_format_as_audio_format() -> None:
+    request: Final = SpeechToCompletionBridgeTransformationHandler().transform_request(
+        model="gpt-4o-audio-preview",
+        input="Hello from LiteLLM",
+        voice="alloy",
+        optional_params={"response_format": "wav"},
+        litellm_params={},
+        headers={},
+        litellm_logging_obj=MagicMock(),
+        custom_llm_provider="openai",
+    )
+
+    assert "response_format" not in request
+    assert request["audio"] == {"voice": "alloy", "format": "wav"}

tests/test_litellm/integrations/test_shadow_eval_logger.py

@@ -58,11 +58,12 @@ def _prisma(jobs=(), attempt_counts=(), attempt_costs=()) -> MagicMock:
     return prisma
 
 
-def _job_record(job: ActiveShadowEvalJob, api_key_id="key-hash") -> MagicMock:
+def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash") -> MagicMock:
     record = MagicMock()
     for field, value in dict(
         id=job.id,
-        api_key_id=api_key_id,
+        target_type=target_type,
+        target_id=target_id,
         router_name=job.router_name,
         direction=job.direction,
         baseline_model=job.baseline_model,
@@ -123,7 +124,7 @@ def _spend_counter(store=None):
     return counter, read, write
 
 
-def _logger(router=None, prisma=None, jobs=(), counter_store=None) -> ShadowEvalLogger:
+def _logger(router=None, prisma=None, jobs=(), counter_store=None, jobs_by_target=None) -> ShadowEvalLogger:
     cache = InMemoryCache(max_size_in_memory=4, default_ttl=60)
     counter, read, write = _spend_counter(counter_store)
     funnel_events = []
@@ -137,8 +138,9 @@ def _logger(router=None, prisma=None, jobs=(), counter_store=None) -> ShadowEval
     )
     logger._test_counter = counter
     logger._test_funnel = funnel_events
-    if jobs:
-        cache.set_cache("shadow_eval:active_jobs", {"key-hash": tuple(jobs)})
+    seeded = jobs_by_target if jobs_by_target is not None else ({("key", "key-hash"): tuple(jobs)} if jobs else None)
+    if seeded is not None:
+        cache.set_cache("shadow_eval:active_jobs", seeded)
     return logger
 
 
@@ -837,6 +839,86 @@ class TestSuccessHookSkipChain:
         prisma.db.litellm_shadowevalattempt.create.assert_not_called()
 
 
+JWT_IDENTITY = {"user_api_key_hash": None, "user_api_key_team_id": "team-eng", "user_api_key_user_id": "dev-alice"}
+
+
+@pytest.mark.asyncio
+class TestTargetMatching:
+    """A request qualifies for a job through ANY of its resolved identities: key hash,
+    team id, or user id. Team and user jobs must therefore sample JWT-authenticated
+    traffic, which carries no key hash at all."""
+
+    @pytest.mark.parametrize(
+        "target,sampled",
+        [
+            (("team", "team-eng"), True),
+            (("user", "dev-alice"), True),
+            (("key", "some-key"), False),
+        ],
+        ids=["team-job-samples-jwt-traffic", "user-job-samples-jwt-traffic", "key-jobs-never-match-keyless-traffic"],
+    )
+    async def test_jwt_shaped_traffic_matches_team_and_user_jobs_but_no_key_job(self, target, sampled):
+        prisma = _prisma()
+        router = _router()
+        logger = _logger(router=router, prisma=prisma, jobs_by_target={target: (_job(),)})
+        hook_kwargs = _success_kwargs()
+        hook_kwargs["standard_logging_object"]["metadata"] = dict(JWT_IDENTITY)
+
+        await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None)
+        await _drain(logger)
+
+        if sampled:
+            prisma.db.litellm_shadowevalattempt.create.assert_awaited_once()
+            assert prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]["job_id"] == "job-1"
+        else:
+            router.acompletion.assert_not_called()
+            prisma.db.litellm_shadowevalattempt.create.assert_not_called()
+
+    async def test_an_event_with_no_identity_early_returns_without_a_cache_read(self):
+        prisma = _prisma()
+        router = _router()
+        cache = MagicMock(spec=InMemoryCache)
+        cache.async_get_cache = AsyncMock()
+        logger = ShadowEvalLogger(
+            router_provider=lambda: router,
+            prisma_provider=lambda: prisma,
+            jobs_cache=cache,
+        )
+        hook_kwargs = _success_kwargs()
+        hook_kwargs["standard_logging_object"]["metadata"] = {}
+
+        await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None)
+
+        cache.async_get_cache.assert_not_awaited()
+        router.acompletion.assert_not_called()
+        prisma.db.litellm_shadowevalattempt.create.assert_not_called()
+
+    async def test_an_event_matching_a_key_job_and_a_team_job_fires_both(self):
+        """A request's key and its team can each hold a job; the two are separately
+        budgeted experiments, so both fire and each counts its own start."""
+        prisma = _prisma()
+        logger = _logger(
+            router=_router(),
+            prisma=prisma,
+            jobs_by_target={
+                ("key", "key-hash"): (_job(id="key-job"),),
+                ("team", "team-eng"): (_job(id="team-job"),),
+            },
+        )
+        hook_kwargs = _success_kwargs()
+        hook_kwargs["standard_logging_object"]["metadata"] = {
+            "user_api_key_hash": "key-hash",
+            "user_api_key_team_id": "team-eng",
+        }
+
+        await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None)
+        await _drain(logger)
+
+        rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.call_args_list]
+        assert sorted(row["job_id"] for row in rows) == ["key-job", "team-job"]
+        assert logger._job_starts == {"key-job": 1, "team-job": 1}
+
+
 @pytest.mark.asyncio
 class TestActiveJobsCache:
     async def test_cache_miss_reads_db_once_then_serves_from_cache(self):
@@ -851,8 +933,8 @@ class TestActiveJobsCache:
         first = await logger._active_jobs()
         second = await logger._active_jobs()
 
-        assert [job.id for job in first["key-hash"]] == ["job-1"]
-        assert second["key-hash"][0].attempts == 7
+        assert [job.id for job in first[("key", "key-hash")]] == ["job-1"]
+        assert second[("key", "key-hash")][0].attempts == 7
         assert prisma.db.litellm_shadowevaljob.find_many.await_count == 1
         where = prisma.db.litellm_shadowevaljob.find_many.call_args.kwargs["where"]
         assert where["stopped_at"] is None
@@ -899,8 +981,8 @@ class TestActiveJobsCache:
         jobs = await logger._active_jobs()
 
         assert logger._job_starts == {}
-        assert jobs["key-hash"][0].attempts == 7
-        assert jobs["key-hash"][0].spend == 0.05
+        assert jobs[("key", "key-hash")][0].attempts == 7
+        assert jobs[("key", "key-hash")][0].spend == 0.05
 
 
 @pytest.mark.asyncio
@@ -1249,13 +1331,14 @@ class TestActiveJobsFailClosed:
             jobs_cache=InMemoryCache(max_size_in_memory=4, default_ttl=60),
         )
 
-        assert [job.id for job in (await logger._active_jobs())["key-hash"]] == ["job-ok"]
+        assert [job.id for job in (await logger._active_jobs())[("key", "key-hash")]] == ["job-ok"]
 
-    async def test_both_of_a_key_s_jobs_survive_the_lookup(self):
+    async def test_every_targets_jobs_survive_the_lookup_keyed_by_type_and_id(self):
         records = [
             _job_record(_job(id="job-forward")),
             _job_record(_reverse_job(id="job-reverse")),
-            _job_record(_job(id="job-other"), api_key_id="other-key"),
+            _job_record(_job(id="job-other"), target_id="other-key"),
+            _job_record(_job(id="job-team"), target_type="team", target_id="team-eng"),
         ]
         prisma = _prisma(jobs=records, attempt_counts=[("job-reverse", 3)])
         logger = ShadowEvalLogger(
@@ -1266,9 +1349,11 @@ class TestActiveJobsFailClosed:
 
         jobs = await logger._active_jobs()
 
-        assert sorted(job.id for job in jobs["key-hash"]) == ["job-forward", "job-reverse"]
-        assert [job.id for job in jobs["other-key"]] == ["job-other"]
-        assert {job.id: job.attempts for job in jobs["key-hash"]}["job-reverse"] == 3
+        assert sorted(job.id for job in jobs[("key", "key-hash")]) == ["job-forward", "job-reverse"]
+        assert [job.id for job in jobs[("key", "other-key")]] == ["job-other"]
+        assert [job.id for job in jobs[("team", "team-eng")]] == ["job-team"]
+        assert ("team-eng",) not in jobs and "team-eng" not in jobs
+        assert {job.id: job.attempts for job in jobs[("key", "key-hash")]}["job-reverse"] == 3
 
 
 def _failing_router():

tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py

@@ -184,3 +184,26 @@ class TestTogetherApiBaseResolvesProvider:
 
         assert provider == "together_ai"
         assert api_base == "https://api.together.ai/v1"
+
+
+class TestGigachatApiBaseResolvesProvider:
+    """
+    Regression for the GigaChat api_base branch: the provider-mapping chain
+    carried an ``endpoint == "https://gigachat.devices.sberbank.ru/api/v1"``
+    elif, but the URL was never added to ``openai_compatible_endpoints``, so
+    the endpoint loop never fired the branch and a caller-supplied GigaChat
+    api_base raised BadRequestError instead of resolving to ``gigachat``.
+    """
+
+    def test_gigachat_api_base_resolves_to_gigachat(self, monkeypatch):
+        monkeypatch.setenv("GIGACHAT_API_KEY", "gigachat-key-from-env")
+
+        model, provider, dynamic_api_key, returned_api_base = get_llm_provider(
+            model="GigaChat-2",
+            api_base="https://gigachat.devices.sberbank.ru/api/v1",
+        )
+
+        assert provider == "gigachat"
+        assert dynamic_api_key == "gigachat-key-from-env"
+        assert returned_api_base == "https://gigachat.devices.sberbank.ru/api/v1"
+        assert model == "GigaChat-2"

tests/test_litellm/litellm_core_utils/test_litellm_logging.py

@@ -6101,3 +6101,64 @@ def test_response_timing_metrics_survive_deepcopy(logging_obj):
     logging_obj.set_response_timing_metrics({"_response_ms": 12.5})
 
     assert copy.deepcopy(logging_obj).response_timing_metrics == {"_response_ms": 12.5}
+
+
+def test_passthrough_embeddings_result_swapped_for_callbacks():
+    """
+    Regression: for gigachat passthrough /embeddings, normalize_logging_result
+    produces an EmbeddingResponse, but the result swap only accepted
+    ModelResponse, so callbacks kept receiving the raw httpx.Response (which
+    crashes attribute readers like OTEL). The swap must cover
+    EmbeddingResponse too.
+    """
+    import datetime as dt
+
+    from litellm.types.utils import EmbeddingResponse
+
+    logging_obj = LitellmLogging(
+        model="EmbeddingsGigaR",
+        messages=[],
+        stream=False,
+        call_type="allm_passthrough_route",
+        start_time=time.time(),
+        litellm_call_id="passthrough-embed-call-id",
+        function_id="passthrough-embed-fn-id",
+    )
+    logging_obj.update_environment_variables(
+        litellm_params={},
+        optional_params={},
+        model="EmbeddingsGigaR",
+        custom_llm_provider="gigachat",
+        endpoint="/embeddings",
+        request_data={"model": "EmbeddingsGigaR", "input": ["hello"]},
+        input=["hello"],
+    )
+
+    httpx_response = httpx.Response(
+        200,
+        json={
+            "object": "list",
+            "data": [
+                {
+                    "object": "embedding",
+                    "embedding": [0.1, 0.2, 0.3],
+                    "index": 0,
+                    "usage": {"prompt_tokens": 5},
+                }
+            ],
+            "model": "EmbeddingsGigaR",
+        },
+        request=httpx.Request(
+            "POST", "https://gigachat.devices.sberbank.ru/api/v1/embeddings"
+        ),
+    )
+
+    _, _, swapped_result = logging_obj._success_handler_helper_fn(
+        result=httpx_response,
+        start_time=dt.datetime.now(),
+        end_time=dt.datetime.now(),
+        cache_hit=False,
+    )
+
+    assert isinstance(swapped_result, EmbeddingResponse)
+    assert swapped_result.data[0]["embedding"] == [0.1, 0.2, 0.3]

tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py

@@ -1490,6 +1490,92 @@ class MockCanaryMaskingGuardrail(CustomGuardrail):
         return inputs
 
 
+class TestAnthropicMessagesImageSources:
+    """An Anthropic image block has three source shapes (`AnthropicMessagesImageParam.source`).
+
+    Only the base64 one carries "data", so reading that key alone drops url images
+    entirely -- for every guardrail consuming GenericGuardrailAPIInputs["images"],
+    not just Bedrock.
+    """
+
+    def _data(self, messages):
+        return {"model": "claude-sonnet-4-5", "messages": messages}
+
+    async def _images_seen(self, content) -> list[str]:
+        handler = AnthropicMessagesHandler()
+
+        class ImageRecordingGuardrail(MockCanaryMaskingGuardrail):
+            def __init__(self):
+                super().__init__()
+                self.seen_images: list[str] = []  # mutable-ok: accumulator for the assertion
+
+            async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+                self.seen_images.extend(inputs.get("images") or [])
+                return await super().apply_guardrail(inputs, request_data, input_type, logging_obj)
+
+        guardrail = ImageRecordingGuardrail()
+        # The text block is what gets the guardrail invoked at all: a message with
+        # no text gives the handler nothing to scan, so it never reaches the
+        # guardrail and every source shape would look equally "dropped".
+        await handler.process_input_messages(
+            data=self._data([{"role": "user", "content": [{"type": "text", "text": "describe it"}, *content]}]),
+            guardrail_to_apply=guardrail,
+        )
+        return guardrail.seen_images
+
+    @pytest.mark.asyncio
+    async def test_url_source_reaches_the_guardrail(self):
+        """A url source has no "data" key, so it used to yield nothing at all."""
+        seen = await self._images_seen(
+            [{"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}]
+        )
+
+        assert seen == ["https://example.com/a.png"]
+
+    @pytest.mark.asyncio
+    async def test_base64_source_carries_its_media_type(self):
+        """Bare base64 leaves the consumer no way to recover the format.
+
+        An API like Bedrock's ApplyGuardrail needs it to build the request, so the
+        media_type travels with the payload as a data URI.
+        """
+        seen = await self._images_seen(
+            [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}}]
+        )
+
+        assert seen == ["data:image/png;base64,AAAA"]
+
+    @pytest.mark.asyncio
+    async def test_base64_source_without_a_media_type_is_passed_through(self):
+        """There is no format to attach, so the payload goes through unchanged."""
+        seen = await self._images_seen([{"type": "image", "source": {"type": "base64", "data": "AAAA"}}])
+
+        assert seen == ["AAAA"]
+
+    @pytest.mark.asyncio
+    async def test_file_source_yields_nothing(self):
+        """The bytes live behind the Files API and this extractor has no client.
+
+        Documented as a known gap rather than silently handed on as a file_id string,
+        which a consumer would try to decode as an image.
+        """
+        seen = await self._images_seen([{"type": "image", "source": {"type": "file", "file_id": "file_abc"}}])
+
+        assert seen == []
+
+    @pytest.mark.asyncio
+    async def test_a_malformed_source_is_dropped_rather_than_passed_on(self):
+        seen = await self._images_seen(
+            [
+                {"type": "image", "source": {"type": "base64"}},
+                {"type": "image", "source": {"type": "url"}},
+                {"type": "image", "source": {"type": "base64", "data": ""}},
+            ]
+        )
+
+        assert seen == []
+
+
 class TestAnthropicMessagesToolResultScanning:
     """LIT-5251: tool_result blocks carry whatever a client's local tool fetched, so
     they are the request-path payload an indirect prompt injection actually arrives in.

tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py

@@ -6,9 +6,7 @@ import pytest
 
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.llms.anthropic.experimental_pass_through.messages import (
-    streaming_iterator as streaming_iterator_module,
-)
+from litellm.llms.anthropic.experimental_pass_through.messages import streaming_iterator as streaming_iterator_module
 from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
     INCOMPLETE_STREAM_ERROR_MESSAGE,
     AnthropicMessagesStreamHiddenParams,
@@ -338,47 +336,6 @@ async def _events_then_hang(events):
     await asyncio.Event().wait()
 
 
-@pytest.mark.asyncio
-async def test_async_sse_wrapper_logs_partial_chunks_on_client_disconnect():
-    """
-    Regression test for LIT-5839: a client disconnect tears the generator
-    down with GeneratorExit at the yield, which used to skip the post-loop
-    logging dispatch entirely, so the partial output tokens the provider
-    already generated (and billed) never reached spend tracking.
-    """
-    iterator = _RecordingLoggingIterator(
-        litellm_logging_obj=_make_logging_obj("test_disconnect_logs_partial_chunks"),
-        request_body={},
-    )
-    wrapped = iterator.async_sse_wrapper(_events_then_hang(TRUNCATED_TOOL_USE_EVENTS))
-    streamed = [await wrapped.__anext__() for _ in range(len(TRUNCATED_TOOL_USE_EVENTS))]
-    assert iterator.logging_call_count == 0
-
-    await wrapped.aclose()
-
-    assert iterator.logging_call_count == 1
-    assert iterator.logged_chunks == streamed
-
-
-@pytest.mark.asyncio
-async def test_async_sse_wrapper_logs_partial_chunks_on_cancellation():
-    iterator = _RecordingLoggingIterator(
-        litellm_logging_obj=_make_logging_obj("test_cancellation_logs_partial_chunks"),
-        request_body={},
-    )
-    wrapped = iterator.async_sse_wrapper(_events_then_hang(TRUNCATED_TOOL_USE_EVENTS))
-    streamed = [await wrapped.__anext__() for _ in range(len(TRUNCATED_TOOL_USE_EVENTS))]
-
-    consume_task = asyncio.ensure_future(wrapped.__anext__())
-    await asyncio.sleep(0.01)
-    consume_task.cancel()
-    with pytest.raises(asyncio.CancelledError):
-        await consume_task
-
-    assert iterator.logging_call_count == 1
-    assert iterator.logged_chunks == streamed
-
-
 @pytest.mark.asyncio
 async def test_async_sse_wrapper_skips_logging_on_disconnect_before_first_chunk():
     iterator = _RecordingLoggingIterator(
@@ -408,6 +365,561 @@ def test_incomplete_stream_error_sse_event_is_valid_anthropic_error():
     assert event.endswith("\n\n")
 
 
+_STREAM_PREFIX = (
+    {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}},
+    {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
+    {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "The Roman"}},
+)
+_STREAM_TAIL = (
+    {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " Empire ..."}},
+    {"type": "content_block_stop", "index": 0},
+    {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 64}},
+    {"type": "message_stop"},
+)
+
+
+def _output_tokens_from_logged_chunks(chunks: list[bytes]) -> int | None:
+    """Read the last output_tokens the billing path would see from the SSE bytes."""
+    latest: int | None = None
+    for raw in chunks:
+        for line in raw.decode().splitlines():
+            if not line.startswith("data:"):
+                continue
+            data = json.loads(line[len("data:"):].strip())
+            usage = data.get("usage") if isinstance(data, dict) else None
+            if isinstance(usage, dict) and usage.get("output_tokens") is not None:
+                latest = usage["output_tokens"]
+    return latest
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_bills_full_stream_after_client_disconnect():
+    """
+    Regression: on a client disconnect mid-stream the upstream provider keeps
+    generating (and billing) the full response. The wrapper must keep draining
+    that upstream to its terminal ``message_delta`` and bill the real
+    output_tokens (64), not the partial count the client drained before leaving
+    (the message_start placeholder, 1).
+
+    A ``tail_gated`` event holds back the stream tail until the client has
+    disconnected, so the tail can only be captured by a drain that survives the
+    client teardown - exactly the path the previous implementation dropped.
+    """
+    tail_gated = asyncio.Event()
+    upstream_fully_drained = asyncio.Event()
+
+    async def _gated_stream():
+        for event in _STREAM_PREFIX:
+            yield event
+        await tail_gated.wait()
+        for event in _STREAM_TAIL:
+            yield event
+        upstream_fully_drained.set()
+
+    iterator = _RecordingLoggingIterator(
+        litellm_logging_obj=_make_logging_obj("test_bills_full_stream_after_disconnect"),
+        request_body={},
+    )
+
+    gen = iterator.async_sse_wrapper(_gated_stream())
+
+    client_chunks = []
+    async for chunk in gen:
+        client_chunks.append(chunk)
+        if len(client_chunks) == len(_STREAM_PREFIX):
+            break
+    await gen.aclose()
+
+    tail_gated.set()
+    await asyncio.wait_for(upstream_fully_drained.wait(), timeout=5)
+    for _ in range(100):
+        if iterator.logged_chunks:
+            break
+        await asyncio.sleep(0.01)
+
+    assert len(client_chunks) == len(_STREAM_PREFIX)
+
+    assert iterator.logged_chunks, "pump never billed after client disconnect"
+    assert _output_tokens_from_logged_chunks(iterator.logged_chunks) == 64
+    assert any(c.startswith(b"event: message_stop\n") for c in iterator.logged_chunks)
+    assert not any(c.startswith(b"event: error\n") for c in iterator.logged_chunks)
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_bills_full_stream_when_client_reads_all():
+    """Happy path: when the client drains the whole stream, billing still sees
+    the terminal output_tokens (64) and the client gets every chunk."""
+    tail_gated = asyncio.Event()
+    tail_gated.set()  # no gating; full stream flows immediately
+
+    async def _full_stream():
+        for event in (*_STREAM_PREFIX, *_STREAM_TAIL):
+            yield event
+
+    iterator = _RecordingLoggingIterator(
+        litellm_logging_obj=_make_logging_obj("test_bills_full_stream_happy_path"),
+        request_body={},
+    )
+    client_chunks = [chunk async for chunk in iterator.async_sse_wrapper(_full_stream())]
+
+    for _ in range(100):
+        if iterator.logged_chunks:
+            break
+        await asyncio.sleep(0.01)
+
+    assert len(client_chunks) == len(_STREAM_PREFIX) + len(_STREAM_TAIL)
+    assert _output_tokens_from_logged_chunks(iterator.logged_chunks) == 64
+    assert not any(c.startswith(b"event: error\n") for c in iterator.logged_chunks)
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_dispatches_deferred_logging_when_client_disconnects_mid_tail():
+    """
+    Regression: when the pump finishes draining while the client is still
+    connected, ``_handle_streaming_logging`` defers billing for the proxy's
+    post-response hook (``ProxyLogging._fire_deferred_stream_logging``), which
+    only fires on a normally completed response. If the client then disconnects
+    before consuming the queued tail, the response generator tears down via
+    GeneratorExit and that hook never runs. The relay teardown must dispatch
+    the stored deferred billing itself, or the request logs no spend at all.
+    """
+    dispatched = []
+    deferred_fired = asyncio.Event()
+
+    def _deferred_stream_complete(logging_coroutine):
+        dispatched.append(logging_coroutine)
+
+        async def _consume():
+            logging_coroutine.close()
+            deferred_fired.set()
+
+        return _consume()
+
+    logging_obj = _make_logging_obj("test_deferred_dispatch_on_disconnect_mid_tail")
+    logging_obj._on_deferred_stream_complete = _deferred_stream_complete
+    iterator = BaseAnthropicMessagesStreamingIterator(litellm_logging_obj=logging_obj, request_body={})
+
+    async def _full_stream():
+        for event in (*_STREAM_PREFIX, *_STREAM_TAIL):
+            yield event
+
+    gen = iterator.async_sse_wrapper(_full_stream())
+    client_chunks = []
+    async for chunk in gen:
+        client_chunks.append(chunk)
+        if len(client_chunks) == len(_STREAM_PREFIX):
+            break
+
+    for _ in range(100):
+        if getattr(logging_obj, "_deferred_stream_complete_args", None) is not None:
+            break
+        await asyncio.sleep(0.01)
+    assert getattr(logging_obj, "_deferred_stream_complete_args", None) is not None, "pump never deferred billing"
+
+    await gen.aclose()
+
+    assert len(dispatched) == 1, "relay teardown did not dispatch the deferred billing"
+    assert logging_obj._on_deferred_stream_complete is None
+    assert logging_obj._deferred_stream_complete_args is None
+    await asyncio.wait_for(deferred_fired.wait(), timeout=5)
+
+
+class _ProviderStreamError(Exception):
+    """Stand-in for a provider-specific streaming failure carrying a status code."""
+
+    def __init__(self, message: str, status_code: int):
+        super().__init__(message)
+        self.status_code = status_code
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_reraises_upstream_error_to_connected_client():
+    """
+    Regression: an upstream failure (Bedrock read / decode / chunk-conversion)
+    before message_stop must propagate the ORIGINAL provider exception to a
+    still-connected client, so the proxy's failure handling keeps the
+    provider-specific status. The pump must not swallow it into a generic
+    api_error event + normal termination.
+    """
+
+    async def _failing_stream():
+        yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}}
+        yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}}
+        raise _ProviderStreamError("bedrock stream blew up", status_code=529)
+
+    iterator = _RecordingLoggingIterator(
+        litellm_logging_obj=_make_logging_obj("test_reraises_upstream_error"),
+        request_body={},
+    )
+
+    received = []
+
+    async def _drain():
+        async for chunk in iterator.async_sse_wrapper(_failing_stream()):
+            received.append(chunk)
+
+    with pytest.raises(_ProviderStreamError) as excinfo:
+        await _drain()
+
+    assert excinfo.value.status_code == 529
+    assert received
+    assert not any(c.startswith(b"event: error\n") for c in received)
+    assert iterator.logged_chunks == []
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_salvages_partial_spend_on_upstream_error_after_disconnect():
+    """
+    When the upstream errors AFTER the client has already disconnected there is
+    no live client to re-raise to and no failure hook will run, so the pump
+    salvages partial spend from what it collected instead of dropping the row.
+    """
+    tail_gated = asyncio.Event()
+
+    async def _gated_failing_stream():
+        yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}}
+        yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}}
+        await tail_gated.wait()
+        raise _ProviderStreamError("late failure", status_code=500)
+
+    iterator = _RecordingLoggingIterator(
+        litellm_logging_obj=_make_logging_obj("test_salvage_partial_on_late_error"),
+        request_body={},
+    )
+
+    gen = iterator.async_sse_wrapper(_gated_failing_stream())
+    received = [await gen.__anext__(), await gen.__anext__()]
+    await gen.aclose()  # client disconnects before the upstream error
+
+    tail_gated.set()  # let the upstream raise now, after disconnect
+    for _ in range(100):
+        if iterator.logged_chunks:
+            break
+        await asyncio.sleep(0.01)
+
+    assert len(received) == 2
+    assert iterator.logged_chunks == received
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_salvages_spend_when_queued_error_is_never_consumed():
+    """
+    When the upstream errors while the client is still connected, the pump
+    forwards the exception through the queue expecting the relay to re-raise it
+    into the proxy's failure handling. If the client disconnects before
+    consuming that queued exception, the handoff never happens and no failure
+    hook runs, so the pump must notice the unconsumed exception at teardown and
+    salvage partial spend instead of dropping the row entirely.
+    """
+    upstream_errored = asyncio.Event()
+
+    async def _failing_stream():
+        yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}}
+        yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}}
+        upstream_errored.set()
+        raise _ProviderStreamError("mid-stream failure", status_code=500)
+
+    iterator = _RecordingLoggingIterator(
+        litellm_logging_obj=_make_logging_obj("test_salvage_on_unconsumed_queued_error"),
+        request_body={},
+    )
+
+    gen = iterator.async_sse_wrapper(_failing_stream())
+    received = [await gen.__anext__(), await gen.__anext__()]
+    await upstream_errored.wait()  # exception is now queued behind the consumed chunks
+    await gen.aclose()  # client disconnects without ever consuming the queued exception
+
+    for _ in range(100):
+        if iterator.logged_chunks:
+            break
+        await asyncio.sleep(0.01)
+
+    assert iterator.logging_call_count == 1
+    assert iterator.logged_chunks == received
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_applies_backpressure_to_slow_client(monkeypatch):
+    """
+    Regression: the relay queue is bounded, so a slow client throttles the
+    upstream read instead of letting the pump buffer the whole response in
+    memory. With a tiny queue and a client that reads a single chunk, the pump
+    must stall after producing only a queue's worth of chunks ahead, not race
+    to the end of a large stream.
+    """
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 2)
+
+    total = 200
+    produced = 0
+
+    async def _fast_stream():
+        nonlocal produced
+        for i in range(total):
+            produced += 1
+            yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"t{i}"}}
+
+    iterator = _make_iterator("test_backpressure_slow_client")
+    gen = iterator.async_sse_wrapper(_fast_stream())
+    try:
+        await gen.__anext__()
+        for _ in range(500):
+            await asyncio.sleep(0)
+        assert produced <= 2 + 3, f"pump ran ahead unthrottled: produced {produced} of {total}"
+        assert produced < total
+    finally:
+        await gen.aclose()
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_bills_partial_when_detached_drain_cap_reached(monkeypatch):
+    """
+    Regression: when the concurrent detached-drain cap is already reached, a
+    pump whose client has disconnected must bill what it collected instead of
+    continuing to drain (and accumulating) the rest of a large upstream stream,
+    so slow/abandoned clients can't pin unbounded worker state.
+
+    The cap slot set is pre-occupied so the single slot is unavailable when this
+    pump reaches its first post-disconnect chunk; that isolates the cap decision
+    from multi-pump scheduling races.
+    """
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 1)
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 4)
+
+    async def _hold_slot():
+        await asyncio.sleep(3600)
+
+    holder = asyncio.ensure_future(_hold_slot())
+    streaming_iterator_module._DETACHED_STREAM_DRAINS.add(holder)
+    tail_reached = False
+
+    async def _long_stream():
+        nonlocal tail_reached
+        yield {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}}
+        yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "x"}}
+        for i in range(100):
+            yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"more{i}"}}
+        tail_reached = True
+        yield {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 42}}
+        yield {"type": "message_stop"}
+
+    iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("drain_cap_full"), request_body={})
+    try:
+        gen = iterator.async_sse_wrapper(_long_stream())
+        await gen.__anext__()  # message_start
+        await gen.__anext__()  # first delta
+        await gen.aclose()  # client disconnects; 100+ chunks remain upstream
+
+        for _ in range(200):
+            if iterator.logged_chunks:
+                break
+            await asyncio.sleep(0)
+
+        assert iterator.logged_chunks, "capped pump never billed"
+        assert len(iterator.logged_chunks) <= 2 + streaming_iterator_module.ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE
+        assert len(iterator.logged_chunks) < 100
+        assert not any(c.startswith(b"event: message_stop\n") for c in iterator.logged_chunks)
+        assert tail_reached is False, "pump kept draining past the cap instead of stopping"
+    finally:
+        holder.cancel()
+        streaming_iterator_module._DETACHED_STREAM_DRAINS.discard(holder)
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_bills_partial_when_detached_drains_disabled(monkeypatch):
+    """
+    Regression: ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS=0 must disable
+    detached draining entirely, not just shrink the cap. With no slots ever
+    available, the very first post-disconnect chunk must fall back to partial
+    spend logging instead of hanging on a cap that's unreachable.
+    """
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 0)
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 4)
+
+    tail_reached = False
+
+    async def _long_stream():
+        nonlocal tail_reached
+        yield {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}}
+        yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "x"}}
+        for i in range(100):
+            yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"more{i}"}}
+        tail_reached = True
+        yield {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 42}}
+        yield {"type": "message_stop"}
+
+    iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("drains_disabled"), request_body={})
+    gen = iterator.async_sse_wrapper(_long_stream())
+    await gen.__anext__()  # message_start
+    await gen.__anext__()  # first delta
+    await gen.aclose()  # client disconnects; 100+ chunks remain upstream
+
+    for _ in range(200):
+        if iterator.logged_chunks:
+            break
+        await asyncio.sleep(0)
+
+    assert iterator.logged_chunks, "pump never billed with detached drains disabled"
+    assert len(iterator.logged_chunks) <= 2 + streaming_iterator_module.ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE
+    assert len(iterator.logged_chunks) < 100
+    assert not any(c.startswith(b"event: message_stop\n") for c in iterator.logged_chunks)
+    assert tail_reached is False, "pump kept draining despite detached drains being disabled"
+    assert len(streaming_iterator_module._DETACHED_STREAM_DRAINS) == 0
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_aborts_upstream_when_detached_drain_cap_reached(monkeypatch):
+    """
+    Regression: when the cap is full and a disconnected pump bails, it must call
+    aclose on the upstream stream so the provider stops generating and billing,
+    not continue running the stream while we record only the partial prefix.
+    """
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 1)
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 4)
+
+    async def _hold_slot():
+        await asyncio.sleep(3600)
+
+    holder = asyncio.ensure_future(_hold_slot())
+    streaming_iterator_module._DETACHED_STREAM_DRAINS.add(holder)
+
+    class _AbortableStream:
+        def __init__(self):
+            self.aclose_called = False
+            self._remaining = iter(
+                (
+                    {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}},
+                    {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "x"}},
+                )
+                + tuple(
+                    {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"t{i}"}}
+                    for i in range(50)
+                )
+            )
+
+        def __aiter__(self):
+            return self
+
+        async def __anext__(self):
+            try:
+                return next(self._remaining)
+            except StopIteration:
+                raise StopAsyncIteration
+
+        async def aclose(self):
+            self.aclose_called = True
+
+    stream = _AbortableStream()
+    iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("abort_upstream_at_cap"), request_body={})
+    try:
+        gen = iterator.async_sse_wrapper(stream)
+        await gen.__anext__()
+        await gen.__anext__()
+        await gen.aclose()
+
+        for _ in range(200):
+            if iterator.logged_chunks:
+                break
+            await asyncio.sleep(0)
+
+        assert iterator.logged_chunks, "capped pump never billed"
+        assert stream.aclose_called, "upstream aclose was not called when the detached-drain cap was reached"
+    finally:
+        holder.cancel()
+        streaming_iterator_module._DETACHED_STREAM_DRAINS.discard(holder)
+
+
+@pytest.mark.asyncio
+async def test_abort_upstream_logs_warning_when_aclose_raises(caplog):
+    """_abort_upstream must swallow and log any exception from aclose()."""
+    import logging
+
+    class _ExplodingStream:
+        def __aiter__(self):
+            return self
+
+        async def __anext__(self):
+            raise StopAsyncIteration
+
+        async def aclose(self):
+            raise RuntimeError("aclose exploded")
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        await BaseAnthropicMessagesStreamingIterator._abort_upstream(_ExplodingStream())
+
+    assert any("abort" in r.message and "RuntimeError" in r.message for r in caplog.records)
+
+
+@pytest.mark.asyncio
+async def test_enqueue_for_client_returns_false_when_already_detached():
+    """_enqueue_for_client must return False immediately (without touching the queue)
+    when client_detached is already set before the call."""
+    from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+        BaseAnthropicMessagesStreamingIterator,
+    )
+
+    queue: asyncio.Queue[bytes | None | BaseException] = asyncio.Queue(maxsize=1)
+    client_detached = asyncio.Event()
+    client_detached.set()
+
+    result = await BaseAnthropicMessagesStreamingIterator._enqueue_for_client(queue, client_detached, b"chunk")
+    assert result is False
+    assert queue.empty()
+
+
+@pytest.mark.asyncio
+async def test_enqueue_for_client_returns_false_when_client_detaches_while_queue_full():
+    """_enqueue_for_client must return False (and cancel the put) when the queue
+    is full and client_detached fires before space becomes available."""
+    from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+        BaseAnthropicMessagesStreamingIterator,
+    )
+
+    queue: asyncio.Queue[bytes | None | BaseException] = asyncio.Queue(maxsize=1)
+    queue.put_nowait(b"already-full")
+
+    client_detached = asyncio.Event()
+
+    async def _set_detached_soon():
+        await asyncio.sleep(0.01)
+        client_detached.set()
+
+    asyncio.create_task(_set_detached_soon())
+    result = await BaseAnthropicMessagesStreamingIterator._enqueue_for_client(queue, client_detached, b"new-chunk")
+    assert result is False
+    assert queue.qsize() == 1
+    assert queue.get_nowait() == b"already-full"
+
+
+@pytest.mark.asyncio
+async def test_async_sse_wrapper_drains_detached_when_cap_available(monkeypatch):
+    """Complement to the cap test: with a slot free, a disconnected pump drains
+    the full upstream and bills the terminal usage, and releases its slot after."""
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 1)
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 4)
+
+    async def _stream():
+        yield {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}}
+        yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "x"}}
+        for i in range(20):
+            yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"m{i}"}}
+        yield {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 42}}
+        yield {"type": "message_stop"}
+
+    iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("drain_cap_free"), request_body={})
+    gen = iterator.async_sse_wrapper(_stream())
+    await gen.__anext__()
+    await gen.__anext__()
+    await gen.aclose()
+
+    for _ in range(300):
+        if iterator.logged_chunks:
+            break
+        await asyncio.sleep(0.01)
+
+    assert any(c.startswith(b"event: message_stop\n") for c in iterator.logged_chunks)
+    assert len(streaming_iterator_module._DETACHED_STREAM_DRAINS) == 0
+
+
 def _decode_sse_events(events: tuple[bytes, ...]) -> list[tuple[str, dict]]:
     decoded = []
     for event in events:
@@ -599,20 +1111,35 @@ async def test_normal_end_with_deferred_dispatch_armed_parks_logging_coroutine(m
 @pytest.mark.asyncio
 async def test_client_disconnect_enqueues_immediately_even_when_deferred_dispatch_armed(monkeypatch):
     """
-    On client disconnect the guardrail end-of-stream scan never runs, so
-    deferral would strand the spend log; the teardown path must keep
-    enqueueing immediately (LIT-5839) even when the deferred callback is armed.
+    Regression: on client disconnect the guardrail end-of-stream scan never
+    runs, so deferral would strand the spend log. The detached pump's
+    post-disconnect bill must bypass the deferred-dispatch park and enqueue
+    immediately (LIT-5839) even when the deferred callback is armed (LIT-6409).
     """
     worker = _RecordingLoggingWorker()
     monkeypatch.setattr(streaming_iterator_module, "GLOBAL_LOGGING_WORKER", worker)
     iterator = _make_iterator("test_disconnect_enqueues_when_armed")
     iterator.litellm_logging_obj._on_deferred_stream_complete = _noop_deferred_dispatch
 
-    wrapped = iterator.async_sse_wrapper(_events_then_hang(TRUNCATED_TOOL_USE_EVENTS))
+    tail_gated = asyncio.Event()
+
+    async def _gated_stream():
+        for event in TRUNCATED_TOOL_USE_EVENTS:
+            yield event
+        await tail_gated.wait()
+        yield {"type": "message_stop"}
+
+    wrapped = iterator.async_sse_wrapper(_gated_stream())
     for _ in range(len(TRUNCATED_TOOL_USE_EVENTS)):
         await wrapped.__anext__()
     await wrapped.aclose()
 
+    tail_gated.set()
+    for _ in range(100):
+        if worker.enqueued:
+            break
+        await asyncio.sleep(0.01)
+
     assert len(worker.enqueued) == 1
     assert getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None) is None
     worker.close_enqueued()
@@ -629,3 +1156,123 @@ async def test_normal_end_without_deferred_dispatch_enqueues_immediately(monkeyp
     assert len(worker.enqueued) == 1
     assert getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None) is None
     worker.close_enqueued()
+
+
+def _backpressured_wrapper(iterator, upstream_exhausted: asyncio.Event):
+    async def _stream():
+        try:
+            for event in COMPLETE_STREAM_EVENTS:
+                yield event
+        finally:
+            upstream_exhausted.set()
+
+    return iterator.async_sse_wrapper(_stream())
+
+
+async def _drain_with_pauses_until_upstream_exhausted(gen, upstream_exhausted: asyncio.Event) -> list:
+    received = []
+    while not upstream_exhausted.is_set():
+        received.append(await gen.__anext__())
+        for _ in range(25):
+            await asyncio.sleep(0)
+        assert len(received) <= len(COMPLETE_STREAM_EVENTS)
+    return received
+
+
+@pytest.mark.asyncio
+async def test_normal_end_parks_deferred_logging_even_when_sentinel_enqueue_backpressured(monkeypatch):
+    """
+    Regression: with a full relay queue at end of stream, the pump suspends
+    while enqueueing the end-of-stream sentinel, and a client that then drains
+    the whole tail tears the relay down (setting ``client_detached``) before
+    the pump resumes. That teardown is a normally completed response, not a
+    disconnect: billing must still park for the proxy's post-response hook
+    (preserving post_call decoration such as guardrail_information) instead of
+    enqueueing immediately through the teardown path.
+    """
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 2)
+    worker = _RecordingLoggingWorker()
+    monkeypatch.setattr(streaming_iterator_module, "GLOBAL_LOGGING_WORKER", worker)
+
+    dispatched = []
+
+    async def _deferred_stream_complete(logging_coroutine):
+        dispatched.append(logging_coroutine)
+        logging_coroutine.close()
+
+    iterator = _make_iterator("test_sentinel_backpressure_normal_end")
+    iterator.litellm_logging_obj._on_deferred_stream_complete = _deferred_stream_complete
+
+    upstream_exhausted = asyncio.Event()
+    gen = _backpressured_wrapper(iterator, upstream_exhausted)
+    received = await _drain_with_pauses_until_upstream_exhausted(gen, upstream_exhausted)
+
+    while True:
+        try:
+            received.append(await gen.__anext__())
+        except StopAsyncIteration:
+            break
+
+    for _ in range(100):
+        if worker.enqueued or getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None):
+            break
+        await asyncio.sleep(0.01)
+
+    assert len(received) == len(COMPLETE_STREAM_EVENTS)
+    assert worker.enqueued == [], "fully delivered stream billed through the teardown path"
+    assert dispatched == []
+    parked = getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None)
+    assert parked is not None, "pump never parked deferred billing"
+    parked[0].close()
+
+
+@pytest.mark.asyncio
+async def test_relay_teardown_dispatches_deferred_billing_when_sentinel_never_consumed(monkeypatch):
+    """
+    Regression: when the pump has parked deferred billing but its end-of-stream
+    sentinel never fits in the full relay queue (the client disconnects without
+    draining the tail), the proxy's post-response hook never fires. Exactly one
+    of the relay teardown or the pump's fallback must dispatch the parked
+    billing, or the request logs no spend at all.
+    """
+    monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 2)
+    worker = _RecordingLoggingWorker()
+    monkeypatch.setattr(streaming_iterator_module, "GLOBAL_LOGGING_WORKER", worker)
+
+    dispatched = []
+    deferred_fired = asyncio.Event()
+
+    def _deferred_stream_complete(logging_coroutine):
+        dispatched.append(logging_coroutine)
+
+        async def _consume():
+            logging_coroutine.close()
+            deferred_fired.set()
+
+        return _consume()
+
+    iterator = _make_iterator("test_sentinel_never_consumed_dispatch")
+    iterator.litellm_logging_obj._on_deferred_stream_complete = _deferred_stream_complete
+
+    upstream_exhausted = asyncio.Event()
+    gen = _backpressured_wrapper(iterator, upstream_exhausted)
+    await _drain_with_pauses_until_upstream_exhausted(gen, upstream_exhausted)
+
+    for _ in range(100):
+        if getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None) is not None:
+            break
+        await asyncio.sleep(0.01)
+
+    await gen.aclose()
+
+    for _ in range(100):
+        if dispatched:
+            break
+        await asyncio.sleep(0.01)
+
+    assert len(dispatched) == 1, "parked billing was never dispatched"
+    assert getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None) is None
+    assert getattr(iterator.litellm_logging_obj, "_on_deferred_stream_complete", None) is None
+    assert len(worker.enqueued) == 1, "teardown billing enqueued alongside the deferred dispatch"
+    await worker.enqueued[0]
+    assert deferred_fired.is_set()

tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py

@@ -3066,17 +3066,21 @@ def test_bedrock_invoke_messages_allows_converted_websearch_function_tool():
 async def test_bedrock_sse_wrapper_dispatches_logging_on_client_disconnect():
     """
     Regression test for LIT-5839: closing the outer bedrock_sse_wrapper
-    mid-stream (what the proxy does on a client disconnect) must close the
-    inner async_sse_wrapper deterministically so the partial-stream logging
-    fires. `completion_start_time` is only stamped on the logging object by
-    that dispatch, so it observing a value proves the whole chain ran.
+    mid-stream (what the proxy does on a client disconnect) must not lose the
+    stream's spend logging. Since the detached-pump relay, the upstream read
+    survives the disconnect and billing fires once the provider stream ends,
+    so the dispatch is awaited after releasing the upstream instead of being
+    observed synchronously at aclose(). `completion_start_time` is only
+    stamped on the logging object by that dispatch, so it observing a value
+    proves the whole chain ran.
     """
     cfg = AmazonAnthropicClaudeMessagesConfig()
+    release_upstream = asyncio.Event()
 
-    async def _hanging_stream():
+    async def _gated_stream():
         yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 25, "output_tokens": 1}}}
         yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}}
-        await asyncio.Event().wait()
+        await release_upstream.wait()
 
     logging_obj = LiteLLMLoggingObj(
         model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0",
@@ -3087,11 +3091,16 @@ async def test_bedrock_sse_wrapper_dispatches_logging_on_client_disconnect():
         litellm_call_id="test_bedrock_sse_wrapper_disconnect_logging",
         function_id="test_bedrock_sse_wrapper_disconnect_logging",
     )
-    wrapped = cfg.bedrock_sse_wrapper(_hanging_stream(), litellm_logging_obj=logging_obj, request_body={})
+    wrapped = cfg.bedrock_sse_wrapper(_gated_stream(), litellm_logging_obj=logging_obj, request_body={})
     await wrapped.__anext__()
     await wrapped.__anext__()
     assert logging_obj.completion_start_time is None
 
     await wrapped.aclose()
+    release_upstream.set()
 
+    for _ in range(500):
+        if logging_obj.completion_start_time is not None:
+            break
+        await asyncio.sleep(0.01)
     assert logging_obj.completion_start_time is not None

tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py

@@ -6,7 +6,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
-
+import litellm
 from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.llms.bedrock.realtime.handler import BedrockRealtime
 from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
@@ -104,12 +104,24 @@ class RealtimeClientWS:
         self.closed = True
 
 
-class ImmediatelyEndingBedrockStream:
-    def __init__(self):
+class ScriptedBedrockReceiver:
+    def __init__(self, payloads):
+        self._payloads = list(payloads)
+
+    async def receive(self):
+        if not self._payloads:
+            return None
+        payload = self._payloads.pop(0)
+        return SimpleNamespace(value=SimpleNamespace(bytes_=payload.encode("utf-8")))
+
+
+class ScriptedBedrockStream:
+    def __init__(self, payloads):
         self.input_stream = FakeInputStream()
+        self._receiver = ScriptedBedrockReceiver(payloads)
 
     async def await_output(self):
-        return (None, EndedBedrockReceiver())
+        return (None, self._receiver)
 
 
 class FakeStaticCredentialsResolver:
@@ -151,7 +163,7 @@ def stub_aws_sdk_client(monkeypatch):
 
         async def invoke_model_with_bidirectional_stream(self, operation_input):
             captured["operation_input"] = operation_input
-            return ImmediatelyEndingBedrockStream()
+            return ScriptedBedrockStream(captured.get("scripted_payloads", []))
 
     package = types.ModuleType("aws_sdk_bedrock_runtime")
     client_module = types.ModuleType("aws_sdk_bedrock_runtime.client")
@@ -271,19 +283,132 @@ class TestBedrockRealtimeHandler:
         assert "sessionEnd" in event_names
         assert stream.input_stream.closed
 
+    @pytest.mark.asyncio
+    async def test_forwarded_events_are_filtered_to_logged_types_for_spend_logging(self):
+        handler = BedrockRealtime()
+        stream = ScriptedBedrockStream(
+            [
+                json.dumps({"event": {"userSpeechStart": {}}}),
+                json.dumps({"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}),
+                json.dumps({"event": {"textOutput": {"content": "Hi"}}}),
+                json.dumps({"event": {"contentEnd": {"stopReason": "END_TURN"}}}),
+            ]
+        )
+        client_ws = RealtimeClientWS()
+
+        logged_events = [
+            event
+            async for event in handler._forward_bedrock_to_client(
+                stream,
+                client_ws,
+                BedrockRealtimeConfig(),
+                "amazon.nova-sonic-v1:0",
+                FakeLogging(),
+                {},
+            )
+        ]
+
+        assert [event["type"] for event in logged_events] == ["response.done"]
+        sent_types = [json.loads(message)["type"] for message in client_ws.sent_to_client]
+        assert "input_audio_buffer.speech_started" in sent_types
+        assert "response.text.delta" in sent_types
+        assert "response.done" in sent_types
+        assert client_ws.closed
+
+    @pytest.mark.asyncio
+    async def test_logged_event_types_star_collects_every_forwarded_event(self, monkeypatch):
+        monkeypatch.setattr(litellm, "logged_real_time_event_types", "*")
+        handler = BedrockRealtime()
+        stream = ScriptedBedrockStream(
+            [
+                json.dumps({"event": {"userSpeechStart": {}}}),
+                json.dumps({"event": {"userSpeechEnd": {}}}),
+            ]
+        )
+        client_ws = RealtimeClientWS()
+
+        logged_events = [
+            event
+            async for event in handler._forward_bedrock_to_client(
+                stream,
+                client_ws,
+                BedrockRealtimeConfig(),
+                "amazon.nova-sonic-v1:0",
+                FakeLogging(),
+                {},
+            )
+        ]
+
+        assert [event["type"] for event in logged_events] == [
+            "input_audio_buffer.speech_started",
+            "input_audio_buffer.speech_stopped",
+        ]
+
+    @pytest.mark.asyncio
+    async def test_trailing_usage_after_last_done_is_dispatched_for_spend(self, stub_aws_sdk_client, monkeypatch):
+        import litellm.llms.bedrock.realtime.handler as handler_module
+
+        dispatched = {}
+
+        class RecordingLogging(FakeLogging):
+            async def dispatch_success_handlers(self, result=None, prefer_async_handlers=False, **kwargs):
+                dispatched["events"] = result
+
+        class RecordingLoggingWorker:
+            def ensure_initialized_and_enqueue(self, coro):
+                dispatched["coro"] = coro
+
+        monkeypatch.setattr(handler_module, "GLOBAL_LOGGING_WORKER", RecordingLoggingWorker())
+        stub_aws_sdk_client["scripted_payloads"] = [
+            json.dumps(
+                {
+                    "event": {
+                        "usageEvent": {
+                            "totalInputTokens": 3,
+                            "totalOutputTokens": 6,
+                            "totalTokens": 9,
+                            "details": {
+                                "total": {
+                                    "input": {"speechTokens": 3, "textTokens": 0},
+                                    "output": {"speechTokens": 0, "textTokens": 6},
+                                }
+                            },
+                        }
+                    }
+                }
+            )
+        ]
+
+        await BedrockRealtime().async_realtime(
+            model="amazon.nova-sonic-v1:0",
+            websocket=RealtimeClientWS(),
+            logging_obj=RecordingLogging(),
+            aws_region_name="us-east-1",
+            aws_access_key_id="k",
+            aws_secret_access_key="s",
+        )
+        await dispatched["coro"]
+
+        assert [event["type"] for event in dispatched["events"]] == ["response.done"]
+        usage = dispatched["events"][0]["response"]["usage"]
+        assert (usage["input_tokens"], usage["output_tokens"], usage["total_tokens"]) == (3, 6, 9)
+        assert usage["input_token_details"] == {"audio_tokens": 3, "text_tokens": 0, "cached_tokens": 0}
+        assert usage["output_token_details"] == {"audio_tokens": 0, "text_tokens": 6}
+
     @pytest.mark.asyncio
     async def test_bedrock_stream_end_closes_client_websocket(self):
         handler = BedrockRealtime()
         client_ws = ClosableClientWS()
 
-        await handler._forward_bedrock_to_client(
+        async for _ in handler._forward_bedrock_to_client(
             EndedBedrockStream(),
             client_ws,
             BedrockRealtimeConfig(),
             "amazon.nova-sonic-v1:0",
             MagicMock(),
             {},
-        )
+        ):
+            pass
 
         assert client_ws.closed
 
@@ -320,9 +445,7 @@ class TestBedrockRealtimeSessionLifecycle:
             [json.dumps({"type": "session.update", "session": {"instructions": "hi", "modalities": ["text"]}})]
         )
 
-        await handler._forward_client_to_bedrock(
-            client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging()
-        )
+        await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging())
 
         acked = [json.loads(message) for message in client_ws.sent_to_client]
         updated = [event for event in acked if event["type"] == "session.updated"]
@@ -334,9 +457,7 @@ class TestBedrockRealtimeSessionLifecycle:
         handler = BedrockRealtime()
         config = BedrockRealtimeConfig()
         stream = FakeBedrockStream()
-        client_ws = DisconnectingClientWS(
-            [json.dumps({"type": "session.update", "session": {"instructions": "hi"}})]
-        )
+        client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})])
 
         await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {})
 

tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py

@@ -827,5 +827,310 @@ class TestBedrockRealtimeSessionEvents:
         assert event["session"]["modalities"] == ["text", "audio"]
 
 
+class TestBedrockRealtimeUserEventsAndUsage:
+    """Regression tests for #38346: USER ASR transcripts, speech boundary events,
+    usage propagation, and duplicate response.created"""
+
+    @staticmethod
+    def _run(config, messages):
+        logging_obj = MagicMock()
+        logging_obj.litellm_trace_id = "trace_123"
+        state = {
+            "session_configuration_request": json.dumps({"configured": True}),
+            "current_output_item_id": None,
+            "current_response_id": None,
+            "current_conversation_id": None,
+            "current_delta_chunks": [],
+            "current_item_chunks": [],
+            "current_delta_type": None,
+        }
+        all_events = []
+        for msg in messages:
+            result = config.transform_realtime_response(
+                json.dumps(msg),
+                "amazon.nova-2-sonic-v1:0",
+                logging_obj,
+                realtime_response_transform_input=dict(state),
+            )
+            all_events.extend(result["response"])
+            state.update(
+                {
+                    "current_output_item_id": result["current_output_item_id"],
+                    "current_response_id": result["current_response_id"],
+                    "current_conversation_id": result["current_conversation_id"],
+                    "current_delta_chunks": result["current_delta_chunks"],
+                    "current_delta_type": result["current_delta_type"],
+                }
+            )
+        return all_events
+
+    def test_user_speech_start_and_stop_events(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}],
+        )
+        assert [e["type"] for e in events] == [
+            "input_audio_buffer.speech_started",
+            "input_audio_buffer.speech_stopped",
+        ]
+        assert all(e["event_id"] and e["item_id"] for e in events)
+        assert events[0]["item_id"] == events[1]["item_id"]
+
+    def test_utterance_lifecycle_shares_one_item_id(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {"event": {"userSpeechStart": {}}},
+                {"event": {"userSpeechEnd": {}}},
+                {
+                    "event": {
+                        "contentStart": {
+                            "role": "USER",
+                            "type": "TEXT",
+                            "additionalModelFields": json.dumps({"generationStage": "FINAL"}),
+                        }
+                    }
+                },
+                {"event": {"textOutput": {"content": "ready"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+            ],
+        )
+        item_ids = {e["item_id"] for e in events if "item_id" in e}
+        assert len(item_ids) == 1
+
+    def test_new_utterance_gets_new_item_id(self):
+        config = BedrockRealtimeConfig()
+        first = self._run(config, [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}])
+        second = self._run(config, [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}])
+        assert first[0]["item_id"] == first[1]["item_id"]
+        assert second[0]["item_id"] == second[1]["item_id"]
+        assert first[0]["item_id"] != second[0]["item_id"]
+
+    def test_user_transcript_emits_input_audio_transcription_events(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {
+                    "event": {
+                        "contentStart": {
+                            "role": "USER",
+                            "type": "TEXT",
+                            "additionalModelFields": json.dumps({"generationStage": "FINAL"}),
+                        }
+                    }
+                },
+                {"event": {"textOutput": {"content": "ready"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+            ],
+        )
+        deltas = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.delta"]
+        completed = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.completed"]
+        assert len(deltas) == 1 and deltas[0]["delta"] == "ready"
+        assert len(completed) == 1 and completed[0]["transcript"] == "ready"
+        assert deltas[0]["item_id"] == completed[0]["item_id"]
+        assert not any(e["type"] == "response.text.delta" for e in events)
+
+    def test_speculative_user_transcript_emits_delta_only(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {
+                    "event": {
+                        "contentStart": {
+                            "role": "USER",
+                            "type": "TEXT",
+                            "additionalModelFields": json.dumps({"generationStage": "SPECULATIVE"}),
+                        }
+                    }
+                },
+                {"event": {"textOutput": {"content": "rea"}}},
+            ],
+        )
+        assert [e["type"] for e in events] == ["conversation.item.input_audio_transcription.delta"]
+
+    def test_user_transcript_state_resets_on_content_end(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {"event": {"contentStart": {"role": "USER", "type": "TEXT"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+                {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}},
+                {"event": {"textOutput": {"content": "Hi there"}}},
+            ],
+        )
+        text_deltas = [e for e in events if e["type"] == "response.text.delta"]
+        assert len(text_deltas) == 1 and text_deltas[0]["delta"] == "Hi there"
+        assert not any(e["type"].startswith("conversation.item.input_audio_transcription") for e in events)
+
+    def test_response_created_emitted_once_per_response(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}},
+                {"event": {"textOutput": {"content": "Hi"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+                {"event": {"contentStart": {"role": "ASSISTANT", "type": "AUDIO"}}},
+            ],
+        )
+        assert sum(1 for e in events if e["type"] == "response.created") == 1
+
+    def test_usage_event_propagates_to_response_done(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {
+                    "event": {
+                        "usageEvent": {
+                            "totalInputTokens": 25,
+                            "totalOutputTokens": 40,
+                            "totalTokens": 65,
+                            "details": {
+                                "total": {
+                                    "input": {"speechTokens": 20, "textTokens": 5},
+                                    "output": {"speechTokens": 30, "textTokens": 10},
+                                }
+                            },
+                        }
+                    }
+                },
+                {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}},
+                {"event": {"textOutput": {"content": "Hi"}}},
+                {"event": {"contentEnd": {"stopReason": "END_TURN"}}},
+            ],
+        )
+        done_events = [e for e in events if e["type"] == "response.done"]
+        assert len(done_events) == 1
+        usage = done_events[0]["response"]["usage"]
+        assert usage["input_tokens"] == 25
+        assert usage["output_tokens"] == 40
+        assert usage["total_tokens"] == 65
+        assert usage["input_token_details"]["audio_tokens"] == 20
+        assert usage["input_token_details"]["text_tokens"] == 5
+        assert usage["output_token_details"]["audio_tokens"] == 30
+        assert usage["output_token_details"]["text_tokens"] == 10
+
+    def test_response_done_without_usage_event_reports_zero_usage(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}},
+                {"event": {"textOutput": {"content": "Hi"}}},
+                {"event": {"contentEnd": {"stopReason": "END_TURN"}}},
+            ],
+        )
+        done_events = [e for e in events if e["type"] == "response.done"]
+        assert len(done_events) == 1
+        usage = done_events[0]["response"]["usage"]
+        assert usage["input_tokens"] == 0
+        assert usage["output_tokens"] == 0
+        assert usage["total_tokens"] == 0
+
+    @staticmethod
+    def _usage_event(total_input, total_output, in_speech, in_text, out_speech, out_text):
+        return {
+            "event": {
+                "usageEvent": {
+                    "totalInputTokens": total_input,
+                    "totalOutputTokens": total_output,
+                    "totalTokens": total_input + total_output,
+                    "details": {
+                        "total": {
+                            "input": {"speechTokens": in_speech, "textTokens": in_text},
+                            "output": {"speechTokens": out_speech, "textTokens": out_text},
+                        }
+                    },
+                }
+            }
+        }
+
+    _ASSISTANT_TURN = (
+        {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}},
+        {"event": {"textOutput": {"content": "Hi"}}},
+        {"event": {"contentEnd": {"stopReason": "END_TURN"}}},
+    )
+
+    def test_multi_turn_usage_reports_per_response_deltas_not_cumulative_totals(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                self._usage_event(25, 40, in_speech=20, in_text=5, out_speech=30, out_text=10),
+                *self._ASSISTANT_TURN,
+                self._usage_event(40, 100, in_speech=30, in_text=10, out_speech=75, out_text=25),
+                *self._ASSISTANT_TURN,
+            ],
+        )
+        usages = [e["response"]["usage"] for e in events if e["type"] == "response.done"]
+        assert len(usages) == 2
+        assert (usages[0]["input_tokens"], usages[0]["output_tokens"], usages[0]["total_tokens"]) == (25, 40, 65)
+        assert (usages[1]["input_tokens"], usages[1]["output_tokens"], usages[1]["total_tokens"]) == (15, 60, 75)
+        assert usages[1]["input_token_details"] == {"audio_tokens": 10, "text_tokens": 5, "cached_tokens": 0}
+        assert usages[1]["output_token_details"] == {"audio_tokens": 45, "text_tokens": 15}
+        assert sum(u["total_tokens"] for u in usages) == 140
+
+    def test_usage_reported_after_last_response_done_flushes_as_logged_only_done(self):
+        config = BedrockRealtimeConfig()
+        self._run(
+            config,
+            [
+                self._usage_event(25, 40, in_speech=20, in_text=5, out_speech=30, out_text=10),
+                *self._ASSISTANT_TURN,
+            ],
+        )
+        assert config.leftover_usage_done_events() == ()
+
+        self._run(config, [self._usage_event(25, 46, in_speech=20, in_text=5, out_speech=30, out_text=16)])
+        leftover = config.leftover_usage_done_events()
+        assert len(leftover) == 1
+        assert leftover[0]["type"] == "response.done"
+        usage = leftover[0]["response"]["usage"]
+        assert (usage["input_tokens"], usage["output_tokens"], usage["total_tokens"]) == (0, 6, 6)
+        assert usage["output_token_details"] == {"audio_tokens": 0, "text_tokens": 6}
+        assert config.leftover_usage_done_events() == ()
+
+    def test_final_transcript_fragments_emit_one_completed_with_full_transcript(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {
+                    "event": {
+                        "contentStart": {
+                            "role": "USER",
+                            "type": "TEXT",
+                            "additionalModelFields": json.dumps({"generationStage": "FINAL"}),
+                        }
+                    }
+                },
+                {"event": {"textOutput": {"content": "What is the "}}},
+                {"event": {"textOutput": {"content": "capital of France?"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+            ],
+        )
+        deltas = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.delta"]
+        completed = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.completed"]
+        assert [d["delta"] for d in deltas] == ["What is the ", "capital of France?"]
+        assert len(completed) == 1
+        assert completed[0]["transcript"] == "What is the capital of France?"
+        assert {e["item_id"] for e in deltas + completed} == {completed[0]["item_id"]}
+
+    def test_speculative_transcript_block_end_emits_no_completed(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {
+                    "event": {
+                        "contentStart": {
+                            "role": "USER",
+                            "type": "TEXT",
+                            "additionalModelFields": json.dumps({"generationStage": "SPECULATIVE"}),
+                        }
+                    }
+                },
+                {"event": {"textOutput": {"content": "rea"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+            ],
+        )
+        assert [e["type"] for e in events] == ["conversation.item.input_audio_transcription.delta"]
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])

tests/test_litellm/llms/gigachat/chat/test_gigachat_chat_streaming.py

@@ -0,0 +1,87 @@
+"""
+Tests for litellm.llms.gigachat.chat.streaming
+"""
+
+from litellm.llms.gigachat.chat.streaming import GigaChatModelResponseIterator
+
+
+def _parse(chunk: dict) -> dict:
+    iterator = GigaChatModelResponseIterator(streaming_response=None, sync_stream=True)
+    return dict(iterator.chunk_parser(chunk=chunk))
+
+
+class TestChunkParserUsage:
+    def test_usage_on_stop_chunk(self):
+        parsed = _parse(
+            {
+                "choices": [{"delta": {"content": ""}, "index": 0, "finish_reason": "stop"}],
+                "usage": {"prompt_tokens": 25, "completion_tokens": 7, "total_tokens": 32},
+            }
+        )
+
+        assert parsed["finish_reason"] == "stop"
+        assert parsed["usage"] is not None
+        assert parsed["usage"]["prompt_tokens"] == 25
+        assert parsed["usage"]["completion_tokens"] == 7
+        assert parsed["usage"]["total_tokens"] == 32
+
+    def test_usage_on_function_call_chunk(self):
+        """Regression: a final chunk ending in function_call still carries usage; it must not be dropped."""
+        parsed = _parse(
+            {
+                "choices": [
+                    {
+                        "delta": {"function_call": {"name": "get_weather", "arguments": {"city": "Moscow"}}},
+                        "index": 0,
+                        "finish_reason": "function_call",
+                    }
+                ],
+                "usage": {"prompt_tokens": 40, "completion_tokens": 12, "total_tokens": 52},
+            }
+        )
+
+        assert parsed["finish_reason"] == "tool_calls"
+        assert parsed["tool_use"] is not None
+        assert parsed["usage"] is not None
+        assert parsed["usage"]["prompt_tokens"] == 40
+        assert parsed["usage"]["completion_tokens"] == 12
+        assert parsed["usage"]["total_tokens"] == 52
+
+    def test_usage_on_length_chunk(self):
+        parsed = _parse(
+            {
+                "choices": [{"delta": {"content": "truncated"}, "index": 0, "finish_reason": "length"}],
+                "usage": {"prompt_tokens": 10, "completion_tokens": 128, "total_tokens": 138},
+            }
+        )
+
+        assert parsed["usage"] is not None
+        assert parsed["usage"]["total_tokens"] == 138
+
+    def test_no_usage_on_interim_chunk(self):
+        parsed = _parse({"choices": [{"delta": {"content": "hello"}, "index": 0, "finish_reason": None}]})
+
+        assert parsed["text"] == "hello"
+        assert parsed["is_finished"] is False
+        assert parsed["usage"] is None
+
+    def test_cache_hit_usage_folds_cached_tokens_back_in(self):
+        """GigaChat reports prompt_tokens and total_tokens after subtracting cached tokens
+        (docs example: prompt_tokens=1, precached_prompt_tokens=37, total_tokens=5), so the
+        OpenAI-convention usage must add them back and surface them as cached_tokens."""
+        parsed = _parse(
+            {
+                "choices": [{"delta": {"content": ""}, "index": 0, "finish_reason": "stop"}],
+                "usage": {
+                    "prompt_tokens": 25,
+                    "completion_tokens": 7,
+                    "total_tokens": 32,
+                    "precached_prompt_tokens": 20,
+                },
+            }
+        )
+
+        assert parsed["usage"] is not None
+        assert parsed["usage"]["prompt_tokens"] == 45
+        assert parsed["usage"]["total_tokens"] == 52
+        assert parsed["usage"]["prompt_tokens_details"]["cached_tokens"] == 20

tests/test_litellm/llms/gigachat/chat/test_gigachat_chat_transformation.py

@@ -0,0 +1,883 @@
+"""
+Unit tests for GigaChat chat transformation.
+
+Tests GigaChatConfig covering get_complete_url, validate_environment,
+get_supported_openai_params, map_openai_params, _convert_tools_to_functions,
+_map_tool_choice, _transform_messages, transform_request, transform_response,
+get_model_response_iterator, and get_error_class.
+"""
+
+import json
+from unittest.mock import MagicMock, patch
+
+import httpx
+import pytest
+
+from litellm.llms.gigachat.chat.transformation import (
+    GigaChatConfig,
+    GigaChatError,
+    is_valid_json,
+)
+from litellm.types.utils import ModelResponse, Usage
+
+TRANSFORM_MODULE = "litellm.llms.gigachat.chat.transformation"
+
+
+def _make_httpx_response(
+    body: dict, status_code: int = 200
+) -> httpx.Response:
+    return httpx.Response(
+        status_code=status_code,
+        headers={"content-type": "application/json"},
+        content=json.dumps(body).encode("utf-8"),
+        request=httpx.Request(
+            "POST",
+            "https://gigachat.devices.sberbank.ru/api/v1/chat/completions",
+        ),
+    )
+
+
+# ---------------------------------------------------------------------------
+# is_valid_json
+# ---------------------------------------------------------------------------
+
+
+class TestIsValidJson:
+    def test_valid_json_object(self):
+        assert is_valid_json('{"key": "value"}') is True
+
+    def test_valid_json_array(self):
+        assert is_valid_json("[1, 2, 3]") is True
+
+    def test_valid_json_string(self):
+        assert is_valid_json('"hello"') is True
+
+    def test_invalid_json(self):
+        assert is_valid_json("{invalid}") is False
+
+    def test_empty_string(self):
+        assert is_valid_json("") is False
+
+
+# ---------------------------------------------------------------------------
+# GigaChatConfig
+# ---------------------------------------------------------------------------
+
+
+class TestGetCompleteUrl:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    def test_uses_api_base_from_param(self):
+        url = self.config.get_complete_url(
+            api_base="https://custom.example.com",
+            api_key=None,
+            model="GigaChat",
+            optional_params={},
+            litellm_params={},
+            stream=False,
+        )
+        assert url == "https://custom.example.com/chat/completions"
+
+    def test_uses_api_base_with_trailing_slash(self):
+        url = self.config.get_complete_url(
+            api_base="https://custom.example.com/",
+            api_key=None,
+            model="GigaChat",
+            optional_params={},
+            litellm_params={},
+            stream=False,
+        )
+        # get_api_base passes the value through without stripping the slash
+        assert url == "https://custom.example.com//chat/completions"
+
+    def test_uses_api_base_from_get_api_base_when_none(self):
+        url = self.config.get_complete_url(
+            api_base=None,
+            api_key=None,
+            model="GigaChat",
+            optional_params={},
+            litellm_params={},
+            stream=False,
+        )
+        assert url.endswith("/chat/completions")
+
+
+class TestValidateEnvironment:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    @patch(f"{TRANSFORM_MODULE}.get_access_token", return_value="test-token")
+    @patch(f"{TRANSFORM_MODULE}.get_secret_str", return_value=None)
+    def test_sets_auth_headers(self, mock_get_secret, mock_get_token):
+        headers: dict = {}
+        result = self.config.validate_environment(
+            headers=headers,
+            model="GigaChat",
+            messages=[{"role": "user", "content": "hi"}],
+            optional_params={},
+            litellm_params={},
+            api_key="creds",
+            api_base="https://api.example.com",
+        )
+        assert result["Authorization"] == "Bearer test-token"
+        assert result["Content-Type"] == "application/json"
+        assert result["Accept"] == "application/json"
+
+    @patch(f"{TRANSFORM_MODULE}.get_access_token", return_value="token")
+    @patch(f"{TRANSFORM_MODULE}.get_secret_str", return_value=None)
+    def test_stores_credentials_and_api_base_for_image_uploads(
+        self, mock_get_secret, mock_get_token
+    ):
+        self.config.validate_environment(
+            headers={},
+            model="GigaChat",
+            messages=[],
+            optional_params={},
+            litellm_params={},
+            api_key="my-creds",
+            api_base="https://my-api.example.com",
+        )
+        assert self.config._current_credentials == "my-creds"
+        assert self.config._current_api_base == "https://my-api.example.com"
+
+    @patch(f"{TRANSFORM_MODULE}.get_access_token", return_value="token")
+    @patch(f"{TRANSFORM_MODULE}.get_secret_str")
+    def test_falls_back_to_env_for_credentials(  # test-quality-ok: mock-echo of internal wiring
+        self, mock_get_secret, mock_get_token
+    ):
+        mock_get_secret.return_value = "env-creds"
+        self.config.validate_environment(
+            headers={},
+            model="GigaChat",
+            messages=[],
+            optional_params={},
+            litellm_params={},
+            api_key=None,
+            api_base=None,
+        )
+        mock_get_secret.assert_any_call("GIGACHAT_CREDENTIALS")  # test-quality-ok: mock-echo of internal wiring
+
+
+class TestGetSupportedOpenAiParams:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    def test_returns_expected_params(self):
+        params = self.config.get_supported_openai_params("GigaChat")
+        expected = [
+            "stream",
+            "temperature",
+            "top_p",
+            "max_tokens",
+            "max_completion_tokens",
+            "stop",
+            "tools",
+            "tool_choice",
+            "functions",
+            "function_call",
+            "response_format",
+        ]
+        assert params == expected
+
+
+class TestMapOpenAiParams:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    def test_stream(self):
+        result = self.config.map_openai_params(
+            non_default_params={"stream": True},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result["stream"] is True
+
+    def test_temperature_zero_maps_to_top_p_zero(self):
+        result = self.config.map_openai_params(
+            non_default_params={"temperature": 0},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result["top_p"] == 0
+        assert "temperature" not in result
+
+    def test_temperature_non_zero(self):
+        result = self.config.map_openai_params(
+            non_default_params={"temperature": 0.7},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result["temperature"] == 0.7
+
+    def test_top_p(self):
+        result = self.config.map_openai_params(
+            non_default_params={"top_p": 0.5},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result["top_p"] == 0.5
+
+    def test_max_tokens(self):
+        result = self.config.map_openai_params(
+            non_default_params={"max_tokens": 100},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result["max_tokens"] == 100
+
+    def test_max_completion_tokens(self):
+        result = self.config.map_openai_params(
+            non_default_params={"max_completion_tokens": 200},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result["max_tokens"] == 200
+
+    def test_stop_is_dropped(self):
+        result = self.config.map_openai_params(
+            non_default_params={"stop": ["\n\n"]},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert "stop" not in result
+
+    def test_tools_converted_to_functions(self):
+        tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_weather",
+                    "description": "Get weather",
+                    "parameters": {"type": "object"},
+                },
+            }
+        ]
+        result = self.config.map_openai_params(
+            non_default_params={"tools": tools},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert "functions" in result
+        assert result["functions"] == [
+            {"name": "get_weather", "description": "Get weather", "parameters": {"type": "object"}}
+        ]
+
+    def test_tool_choice_auto(self):
+        result = self.config.map_openai_params(
+            non_default_params={"tool_choice": "auto"},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result.get("function_call") == "auto"
+
+    def test_tool_choice_none(self):
+        result = self.config.map_openai_params(
+            non_default_params={"tool_choice": "none"},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result.get("function_call") == "none"
+
+    def test_tool_choice_required(self):
+        result = self.config.map_openai_params(
+            non_default_params={"tool_choice": "required"},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result.get("function_call") == "auto"
+
+    def test_tool_choice_dict(self):
+        result = self.config.map_openai_params(
+            non_default_params={
+                "tool_choice": {
+                    "type": "function",
+                    "function": {"name": "get_weather"},
+                }
+            },
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result.get("function_call") == {"name": "get_weather"}
+
+    def test_functions(self):
+        funcs = [{"name": "my_func", "description": "desc", "parameters": {}}]
+        result = self.config.map_openai_params(
+            non_default_params={"functions": funcs},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result["functions"] == funcs
+
+    def test_function_call(self):
+        result = self.config.map_openai_params(
+            non_default_params={"function_call": {"name": "my_func"}},
+            optional_params={},
+            model="GigaChat",
+            drop_params=False,
+        )
+        assert result["function_call"] == {"name": "my_func"}
+
+    def test_response_format_json_schema(self):
+        response_format = {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "test_schema",
+                "schema": {"type": "object", "properties": {"name": {"type": "string"}}},
+            },
+        }
+        result = self.config.map_openai_params(
+            non_default_params={"response_format": response_format},
+            optional_params={"functions": []},
+            model="GigaChat",
+            drop_params=False,
+        )
+        # Should add a function for the schema
+        assert len(result["functions"]) == 1
+        assert result["functions"][0]["name"] == "test_schema"
+        assert result["function_call"] == {"name": "test_schema"}
+        assert result["_structured_output"] is True
+
+
+class TestConvertToolsToFunctions:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    def test_converts_function_tools_only(self):
+        tools = [
+            {"type": "function", "function": {"name": "a", "description": "d", "parameters": {}}},
+            {"type": "code_interpreter"},  # should be ignored
+        ]
+        result = self.config._convert_tools_to_functions(tools)
+        assert len(result) == 1
+        assert result[0]["name"] == "a"
+
+    def test_empty_tools(self):
+        assert self.config._convert_tools_to_functions([]) == []
+
+
+class TestMapToolChoice:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    def test_none(self):
+        assert self.config._map_tool_choice("none") == "none"
+
+    def test_auto(self):
+        assert self.config._map_tool_choice("auto") == "auto"
+
+    def test_required(self):
+        assert self.config._map_tool_choice("required") == "auto"
+
+    def test_dict_with_function(self):
+        result = self.config._map_tool_choice(
+            {"type": "function", "function": {"name": "get_weather"}}
+        )
+        assert result == {"name": "get_weather"}
+
+    def test_dict_without_name(self):
+        result = self.config._map_tool_choice(
+            {"type": "function", "function": {}}
+        )
+        assert result is None
+
+    def test_unknown_value(self):
+        assert self.config._map_tool_choice("unknown") is None
+
+
+class TestTransformMessages:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    def test_developer_role_to_system(self):
+        result = self.config._transform_messages(
+            [{"role": "developer", "content": "be helpful"}]
+        )
+        assert result[0]["role"] == "system"
+        assert result[0]["content"] == "be helpful"
+
+    def test_system_message_not_first_becomes_user(self):
+        result = self.config._transform_messages([
+            {"role": "user", "content": "hi"},
+            {"role": "system", "content": "instruction"},
+        ])
+        assert result[0]["role"] == "user"
+        assert result[1]["role"] == "user"
+        assert result[1]["content"] == "instruction"
+
+    def test_tool_role_to_function(self):
+        result = self.config._transform_messages([
+            {"role": "tool", "content": '{"result": "ok"}'}
+        ])
+        assert result[0]["role"] == "function"
+
+    def test_tool_role_content_wraps_non_json(self):
+        result = self.config._transform_messages([
+            {"role": "tool", "content": "plain text"}
+        ])
+        assert result[0]["role"] == "function"
+        assert is_valid_json(result[0]["content"])
+
+    def test_none_content_becomes_empty_string(self):
+        result = self.config._transform_messages([
+            {"role": "user", "content": None}
+        ])
+        assert result[0]["content"] == ""
+
+    def test_name_field_removed(self):
+        result = self.config._transform_messages([
+            {"role": "user", "content": "hi", "name": "John"}
+        ])
+        assert "name" not in result[0]
+
+    def test_tool_calls_converted_to_function_call(self):
+        result = self.config._transform_messages([
+            {
+                "role": "assistant",
+                "content": "",
+                "tool_calls": [
+                    {
+                        "id": "call_abc",
+                        "type": "function",
+                        "function": {
+                            "name": "get_weather",
+                            "arguments": '{"city": "London"}',
+                        },
+                    }
+                ],
+            }
+        ])
+        assert "tool_calls" not in result[0]
+        assert result[0]["function_call"]["name"] == "get_weather"
+        assert result[0]["function_call"]["arguments"] == {"city": "London"}
+
+    def test_tool_calls_with_dict_arguments(self):
+        result = self.config._transform_messages([
+            {
+                "role": "assistant",
+                "content": "",
+                "tool_calls": [
+                    {
+                        "id": "call_xyz",
+                        "type": "function",
+                        "function": {
+                            "name": "search",
+                            "arguments": {"query": "test"},
+                        },
+                    }
+                ],
+            }
+        ])
+        assert result[0]["function_call"]["arguments"] == {"query": "test"}
+
+    def test_list_content_multimodal(self):
+        content = [
+            {"type": "text", "text": "describe this"},
+            {
+                "type": "image_url",
+                "image_url": {"url": "https://example.com/img.jpg"},
+            },
+        ]
+        with patch.object(self.config, "_upload_image", return_value="file-123"):
+            result = self.config._transform_messages([
+                {"role": "user", "content": content}
+            ])
+        assert result[0]["content"] == "describe this"
+        assert result[0]["attachments"] == ["file-123"]
+
+    def test_list_content_with_image_url_string(self):
+        content = [
+            {"type": "text", "text": "look"},
+            {"type": "image_url", "image_url": "https://example.com/img.jpg"},
+        ]
+        with patch.object(self.config, "_upload_image", return_value="file-456"):
+            result = self.config._transform_messages([
+                {"role": "user", "content": content}
+            ])
+        assert result[0]["content"] == "look"
+        assert "file-456" in result[0]["attachments"]
+
+
+class TestTransformRequest:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    def test_builds_basic_request(self):
+        body = self.config.transform_request(
+            model="gigachat/GigaChat",
+            messages=[{"role": "user", "content": "hi"}],
+            optional_params={},
+            litellm_params={},
+            headers={},
+        )
+        assert body["model"] == "GigaChat"
+        assert len(body["messages"]) == 1
+        assert body["messages"][0]["content"] == "hi"
+
+    def test_model_prefix_stripped(self):
+        body = self.config.transform_request(
+            model="gigachat/GigaChat-Pro",
+            messages=[{"role": "user", "content": "hello"}],
+            optional_params={},
+            litellm_params={},
+            headers={},
+        )
+        assert body["model"] == "GigaChat-Pro"
+
+    def test_includes_optional_params(self):
+        body = self.config.transform_request(
+            model="gigachat/GigaChat",
+            messages=[{"role": "user", "content": "hi"}],
+            optional_params={
+                "temperature": 0.5,
+                "max_tokens": 100,
+                "stream": True,
+            },
+            litellm_params={},
+            headers={},
+        )
+        assert body["temperature"] == 0.5
+        assert body["max_tokens"] == 100
+        assert body["stream"] is True
+
+    def test_includes_functions(self):
+        body = self.config.transform_request(
+            model="gigachat/GigaChat",
+            messages=[{"role": "user", "content": "hi"}],
+            optional_params={
+                "functions": [{"name": "my_func"}],
+                "function_call": {"name": "my_func"},
+            },
+            litellm_params={},
+            headers={},
+        )
+        assert body["functions"] == [{"name": "my_func"}]
+        assert body["function_call"] == {"name": "my_func"}
+
+    def test_skips_unsupported_params(self):
+        body = self.config.transform_request(
+            model="gigachat/GigaChat",
+            messages=[{"role": "user", "content": "hi"}],
+            optional_params={"n": 2, "user": "abc"},
+            litellm_params={},
+            headers={},
+        )
+        assert "n" not in body
+        assert "user" not in body
+
+
+class TestTransformResponse:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    def test_basic_response(self):
+        raw = _make_httpx_response({
+            "id": "chatcmpl-123",
+            "created": 1700000000,
+            "model": "GigaChat",
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {"role": "assistant", "content": "Hello!"},
+                    "finish_reason": "stop",
+                }
+            ],
+            "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
+        })
+        model_response = ModelResponse()
+        result = self.config.transform_response(
+            model="gigachat/GigaChat",
+            raw_response=raw,
+            model_response=model_response,
+            logging_obj=MagicMock(),
+            request_data={},
+            messages=[],
+            optional_params={},
+            litellm_params={},
+            encoding=None,
+        )
+        assert result.choices[0].message.content == "Hello!"
+        assert result.choices[0].finish_reason == "stop"
+        assert result.usage.prompt_tokens == 5
+        assert result.usage.total_tokens == 8
+
+    def test_function_call_into_tool_calls(self):
+        raw = _make_httpx_response({
+            "id": "chatcmpl-456",
+            "created": 1700000000,
+            "model": "GigaChat",
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "content": None,
+                        "function_call": {
+                            "name": "get_weather",
+                            "arguments": {"city": "Moscow"},
+                        },
+                    },
+                    "finish_reason": "function_call",
+                }
+            ],
+            "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+        })
+        model_response = ModelResponse()
+        result = self.config.transform_response(
+            model="gigachat/GigaChat",
+            raw_response=raw,
+            model_response=model_response,
+            logging_obj=MagicMock(),
+            request_data={},
+            messages=[],
+            optional_params={},
+            litellm_params={},
+            encoding=None,
+        )
+        assert result.choices[0].finish_reason == "tool_calls"
+        tool_calls = result.choices[0].message.tool_calls
+        assert tool_calls is not None
+        assert len(tool_calls) == 1
+        assert tool_calls[0].function.name == "get_weather"
+        assert '{"city": "Moscow"}' in tool_calls[0].function.arguments
+
+    def test_function_call_structured_output(self):
+        raw = _make_httpx_response({
+            "id": "chatcmpl-789",
+            "created": 1700000000,
+            "model": "GigaChat",
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "content": None,
+                        "function_call": {
+                            "name": "test_schema",
+                            "arguments": {"name": "John"},
+                        },
+                    },
+                    "finish_reason": "function_call",
+                }
+            ],
+            "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+        })
+        model_response = ModelResponse()
+        result = self.config.transform_response(
+            model="gigachat/GigaChat",
+            raw_response=raw,
+            model_response=model_response,
+            logging_obj=MagicMock(),
+            request_data={},
+            messages=[],
+            optional_params={"_structured_output": True},
+            litellm_params={},
+            encoding=None,
+        )
+        # Structured output: function_call -> content
+        assert result.choices[0].finish_reason == "stop"
+        assert result.choices[0].message.content is not None
+        assert '"name": "John"' in result.choices[0].message.content
+
+    def test_function_call_string_arguments(self):
+        raw = _make_httpx_response({
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "function_call": {
+                            "name": "get_weather",
+                            "arguments": '{"city": "Moscow"}',
+                        },
+                    },
+                    "finish_reason": "function_call",
+                }
+            ],
+            "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+        })
+        model_response = ModelResponse()
+        result = self.config.transform_response(
+            model="gigachat/GigaChat",
+            raw_response=raw,
+            model_response=model_response,
+            logging_obj=MagicMock(),
+            request_data={},
+            messages=[],
+            optional_params={},
+            litellm_params={},
+            encoding=None,
+        )
+        tc = result.choices[0].message.tool_calls[0]
+        assert '{"city": "Moscow"}' in tc.function.arguments
+
+    def test_cleans_up_gigachat_specific_fields(self):
+        raw = _make_httpx_response({
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "content": "done",
+                        "functions_state_id": "some-state",
+                    },
+                    "finish_reason": "stop",
+                }
+            ],
+            "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+        })
+        model_response = ModelResponse()
+        result = self.config.transform_response(
+            model="gigachat/GigaChat",
+            raw_response=raw,
+            model_response=model_response,
+            logging_obj=MagicMock(),
+            request_data={},
+            messages=[],
+            optional_params={},
+            litellm_params={},
+            encoding=None,
+        )
+        # functions_state_id should have been removed from the message data
+        assert result.choices[0].message.content == "done"
+
+    def test_raises_on_invalid_json(self):
+        raw = httpx.Response(
+            status_code=500,
+            headers={"content-type": "text/plain"},
+            content=b"not json",
+            request=httpx.Request("POST", "https://example.com"),
+        )
+        model_response = ModelResponse()
+        with pytest.raises(GigaChatError) as exc_info:
+            self.config.transform_response(
+                model="gigachat/GigaChat",
+                raw_response=raw,
+                model_response=model_response,
+                logging_obj=MagicMock(),
+                request_data={},
+                messages=[],
+                optional_params={},
+                litellm_params={},
+                encoding=None,
+            )
+        assert "Invalid JSON response" in str(exc_info.value.message)
+
+    def test_empty_choices(self):
+        raw = _make_httpx_response({
+            "choices": [],
+            "usage": {},
+        })
+        model_response = ModelResponse()
+        result = self.config.transform_response(
+            model="gigachat/GigaChat",
+            raw_response=raw,
+            model_response=model_response,
+            logging_obj=MagicMock(),
+            request_data={},
+            messages=[],
+            optional_params={},
+            litellm_params={},
+            encoding=None,
+        )
+        assert result.choices == []
+
+    def test_function_call_with_non_dict_arguments(self):
+        raw = _make_httpx_response({
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "function_call": {
+                            "name": "say_hello",
+                            "arguments": "hello",
+                        },
+                    },
+                    "finish_reason": "function_call",
+                }
+            ],
+            "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+        })
+        model_response = ModelResponse()
+        result = self.config.transform_response(
+            model="gigachat/GigaChat",
+            raw_response=raw,
+            model_response=model_response,
+            logging_obj=MagicMock(),
+            request_data={},
+            messages=[],
+            optional_params={},
+            litellm_params={},
+            encoding=None,
+        )
+        tc = result.choices[0].message.tool_calls[0]
+        assert tc.function.arguments == "hello"
+
+
+class TestGetModelResponseIterator:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    def test_returns_gigachat_iterator_sync(self):
+        from litellm.llms.gigachat.chat.streaming import (
+            GigaChatModelResponseIterator,
+        )
+
+        result = self.config.get_model_response_iterator(
+            streaming_response=iter(["data"]),
+            sync_stream=True,
+            json_mode=False,
+        )
+        assert isinstance(result, GigaChatModelResponseIterator)
+
+
+class TestGetErrorClass:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    def test_returns_gigachat_error(self):
+        error = self.config.get_error_class(
+            error_message="something went wrong",
+            status_code=400,
+            headers={"x-request-id": "abc"},
+        )
+        assert isinstance(error, GigaChatError)
+        assert error.status_code == 400
+        assert error.message == "something went wrong"
+        assert error.headers == {"x-request-id": "abc"}
+
+
+class TestUploadImage:
+    def setup_method(self):
+        self.config = GigaChatConfig()
+
+    @patch(f"{TRANSFORM_MODULE}.upload_file_sync", return_value="file-uploaded")
+    def test_upload_image_success(self, mock_upload):
+        self.config._current_credentials = "creds"
+        self.config._current_api_base = "https://api.example.com"
+        result = self.config._upload_image("https://example.com/img.jpg")
+        assert result == "file-uploaded"
+        mock_upload.assert_called_once_with(
+            image_url="https://example.com/img.jpg",
+            credentials="creds",
+            api_base="https://api.example.com",
+        )
+
+    @patch(f"{TRANSFORM_MODULE}.upload_file_sync", side_effect=Exception("fail"))
+    def test_upload_image_failure_returns_none(self, mock_upload):
+        result = self.config._upload_image("https://example.com/img.jpg")
+        assert result is None