merge litellm_internal_staging into litellm_lit5602_non_inference_logging

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

README.md

@@ -292,6 +292,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 | [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Cohere (`cohere`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |
 | [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |

litellm/__init__.py

@@ -453,6 +453,7 @@ max_end_user_budget_id: Optional[str] = None
 # backwards compatibility — arbitrary client-supplied identifiers still
 # pass through unchanged.
 validate_end_user_id_in_db: bool = False
+block_requests_for_models_without_pricing: bool = False
 disable_end_user_cost_tracking: Optional[bool] = None
 disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
 enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None

litellm/caching/_embedding_router.py

@@ -16,6 +16,7 @@ from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any, Final
 
 import litellm
+from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
 
 if TYPE_CHECKING:
     from litellm.router import Router
@@ -60,6 +61,13 @@ def resolve_embedding_max_input_tokens(
     return deployment_max_input_tokens
 
 
+def resolve_embedding_timeout(configured_timeout: float | None) -> float:
+    """Explicit cache setting first, else the short semantic-cache default."""
+    if configured_timeout is not None:
+        return configured_timeout
+    return SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
+
+
 def truncate_embedding_input(prompt: str, embedding_model: str, max_input_tokens: int | None) -> str:
     """Keep only the first ``max_input_tokens`` tokens of ``prompt`` for the embedding call."""
     if max_input_tokens is None:

litellm/caching/caching.py

@@ -98,6 +98,7 @@ class Cache:
         qdrant_semantic_cache_embedding_model: str = "text-embedding-ada-002",
         qdrant_semantic_cache_vector_size: int | None = None,
         semantic_cache_embedding_max_input_tokens: int | None = None,
+        semantic_cache_embedding_timeout: float | None = None,
         # GCP IAM authentication parameters
         gcp_service_account: str | None = None,
         gcp_ssl_ca_certs: str | None = None,
@@ -124,6 +125,7 @@ class Cache:
             qdrant_collection_name (str, optional): The name for your qdrant collection. Required if type is "qdrant-semantic".
             similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
             semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
+            semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
 
             # Disk Cache Args
             disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@@ -195,6 +197,7 @@ class Cache:
                 embedding_model=redis_semantic_cache_embedding_model,
                 index_name=redis_semantic_cache_index_name,
                 embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
+                embedding_timeout=semantic_cache_embedding_timeout,
                 **kwargs,
             )
         elif type == LiteLLMCacheType.VALKEY_SEMANTIC:
@@ -211,6 +214,7 @@ class Cache:
                 index_name=valkey_semantic_cache_index_name,
                 startup_nodes=redis_startup_nodes,
                 embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
+                embedding_timeout=semantic_cache_embedding_timeout,
                 **kwargs,
             )
         elif type == LiteLLMCacheType.QDRANT_SEMANTIC:
@@ -223,6 +227,7 @@ class Cache:
                 embedding_model=qdrant_semantic_cache_embedding_model,
                 vector_size=qdrant_semantic_cache_vector_size,
                 embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
+                embedding_timeout=semantic_cache_embedding_timeout,
             )
         elif type == LiteLLMCacheType.LOCAL:
             self.cache = InMemoryCache()

litellm/caching/qdrant_semantic_cache.py

@@ -16,7 +16,11 @@ from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import print_verbose
-from litellm.constants import QDRANT_SCALAR_QUANTILE, QDRANT_VECTOR_SIZE
+from litellm.constants import (
+    QDRANT_SCALAR_QUANTILE,
+    QDRANT_VECTOR_SIZE,
+    SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS,
+)
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_str_from_messages,
 )
@@ -26,6 +30,7 @@ from ._embedding_router import (
     build_router_embedding_metadata,
     resolve_embedding_max_input_tokens,
     resolve_embedding_router,
+    resolve_embedding_timeout,
     truncate_embedding_input,
 )
 from .base_cache import BaseCache
@@ -37,6 +42,7 @@ if TYPE_CHECKING:
 class QdrantSemanticCache(BaseCache):
     CACHE_KEY_FIELD_NAME = "litellm_cache_key"
     embedding_max_input_tokens: int | None = None
+    embedding_timeout: float = SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
 
     def __init__(
         self,
@@ -49,6 +55,7 @@ class QdrantSemanticCache(BaseCache):
         host_type=None,
         vector_size=None,
         embedding_max_input_tokens: int | None = None,
+        embedding_timeout: float | None = None,
     ):
         from litellm.llms.custom_httpx.http_handler import (
             _get_httpx_client,
@@ -68,6 +75,7 @@ class QdrantSemanticCache(BaseCache):
         self.similarity_threshold = similarity_threshold
         self.embedding_model = embedding_model
         self.embedding_max_input_tokens = embedding_max_input_tokens
+        self.embedding_timeout = resolve_embedding_timeout(embedding_timeout)
         self.vector_size = vector_size if vector_size is not None else QDRANT_VECTOR_SIZE
         headers = {}
 
@@ -222,11 +230,15 @@ class QdrantSemanticCache(BaseCache):
                 input=embedding_input,
                 cache={"no-store": True, "no-cache": True},
                 metadata=build_router_embedding_metadata(metadata),
+                timeout=self.embedding_timeout,
+                num_retries=0,
             )
         return litellm.embedding(
             model=self.embedding_model,
             input=embedding_input,
             cache={"no-store": True, "no-cache": True},
+            timeout=self.embedding_timeout,
+            num_retries=0,
         )
 
     async def _get_async_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> EmbeddingResponse:
@@ -238,19 +250,25 @@ class QdrantSemanticCache(BaseCache):
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
         embedding_input: Final = self._embedding_input(prompt, router)
-        if router is not None:
-            return await router.aembedding(
+        embedding_call: Final = (
+            router.aembedding(
                 model=self.embedding_model,
                 input=embedding_input,
                 cache={"no-store": True, "no-cache": True},
                 metadata=build_router_embedding_metadata(metadata),
+                timeout=self.embedding_timeout,
+                num_retries=0,
+            )
+            if router is not None
+            else litellm.aembedding(
+                model=self.embedding_model,
+                input=embedding_input,
+                cache={"no-store": True, "no-cache": True},
+                timeout=self.embedding_timeout,
+                num_retries=0,
             )
-
-        return await litellm.aembedding(
-            model=self.embedding_model,
-            input=embedding_input,
-            cache={"no-store": True, "no-cache": True},
         )
+        return await asyncio.wait_for(embedding_call, self.embedding_timeout)
 
     def set_cache(self, key, value, **kwargs):
         print_verbose(f"qdrant semantic-cache set_cache, kwargs: {kwargs}")

litellm/caching/redis_semantic_cache.py

@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
+from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_str_from_messages,
 )
@@ -27,6 +28,7 @@ from ._embedding_router import (
     build_router_embedding_metadata,
     resolve_embedding_max_input_tokens,
     resolve_embedding_router,
+    resolve_embedding_timeout,
     truncate_embedding_input,
 )
 from .base_cache import BaseCache
@@ -47,6 +49,7 @@ class RedisSemanticCache(BaseCache):
     DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index"
     CACHE_KEY_FIELD_NAME: str = "litellm_cache_key"
     embedding_max_input_tokens: int | None = None
+    embedding_timeout: float = SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
 
     def __init__(
         self,
@@ -58,6 +61,7 @@ class RedisSemanticCache(BaseCache):
         embedding_model: str = "text-embedding-ada-002",
         index_name: str | None = None,
         embedding_max_input_tokens: int | None = None,
+        embedding_timeout: float | None = None,
         **kwargs: object,
     ):
         """
@@ -74,6 +78,8 @@ class RedisSemanticCache(BaseCache):
             index_name: Name for the Redis index
             embedding_max_input_tokens: Truncate prompts to this many tokens before
                 embedding; defaults to the Router deployment's configured max_input_tokens
+            embedding_timeout: Seconds a cache lookup may spend embedding the prompt before it
+                gives up and lets the request continue to the LLM
             ttl: Default time-to-live for cache entries in seconds
             **kwargs: Additional arguments passed to the Redis client
 
@@ -99,6 +105,7 @@ class RedisSemanticCache(BaseCache):
         self.distance_threshold = 1 - similarity_threshold
         self.embedding_model = embedding_model
         self.embedding_max_input_tokens = embedding_max_input_tokens
+        self.embedding_timeout = resolve_embedding_timeout(embedding_timeout)
 
         # Set up Redis connection
         if redis_url is None:
@@ -349,6 +356,8 @@ class RedisSemanticCache(BaseCache):
                     input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
                     metadata=build_router_embedding_metadata(metadata),
+                    timeout=self.embedding_timeout,
+                    num_retries=0,
                 ),
             )
         else:
@@ -358,6 +367,8 @@ class RedisSemanticCache(BaseCache):
                     model=self.embedding_model,
                     input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
+                    timeout=self.embedding_timeout,
+                    num_retries=0,
                 ),
             )
         return embedding_response["data"][0]["embedding"]
@@ -512,20 +523,26 @@ class RedisSemanticCache(BaseCache):
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
         embedding_input: Final = self._embedding_input(prompt, router)
+        embedding_call: Final = (
+            router.aembedding(
+                model=self.embedding_model,
+                input=embedding_input,
+                cache={"no-store": True, "no-cache": True},
+                metadata=build_router_embedding_metadata(metadata),
+                timeout=self.embedding_timeout,
+                num_retries=0,
+            )
+            if router is not None
+            else litellm.aembedding(
+                model=self.embedding_model,
+                input=embedding_input,
+                cache={"no-store": True, "no-cache": True},
+                timeout=self.embedding_timeout,
+                num_retries=0,
+            )
+        )
         try:
-            if router is not None:
-                embedding_response = await router.aembedding(
-                    model=self.embedding_model,
-                    input=embedding_input,
-                    cache={"no-store": True, "no-cache": True},
-                    metadata=build_router_embedding_metadata(metadata),
-                )
-            else:
-                embedding_response = await litellm.aembedding(
-                    model=self.embedding_model,
-                    input=embedding_input,
-                    cache={"no-store": True, "no-cache": True},
-                )
+            embedding_response: Final = await asyncio.wait_for(embedding_call, self.embedding_timeout)
             return embedding_response["data"][0]["embedding"]
         except Exception as e:
             print_verbose(f"Error generating async embedding: {e}")

litellm/caching/valkey_semantic_cache.py

@@ -30,6 +30,7 @@ from litellm._logging import print_verbose
 from litellm._uuid import uuid
 from litellm.llms.valkey.common_utils import build_valkey_url, pack_vector
 
+from ._embedding_router import resolve_embedding_timeout
 from .redis_semantic_cache import RedisSemanticCache
 
 
@@ -62,6 +63,7 @@ class ValkeySemanticCache(RedisSemanticCache):
         sync_client: Redis | None = None,
         async_client: AsyncRedis | None = None,
         embedding_max_input_tokens: int | None = None,
+        embedding_timeout: float | None = None,
         **kwargs: Any,
     ):
         if similarity_threshold is None:
@@ -80,6 +82,7 @@ class ValkeySemanticCache(RedisSemanticCache):
         self.similarity_threshold = similarity_threshold
         self.embedding_model = embedding_model
         self.embedding_max_input_tokens = embedding_max_input_tokens
+        self.embedding_timeout = resolve_embedding_timeout(embedding_timeout)
         self.index_name = index_name or self.DEFAULT_VALKEY_INDEX_NAME
         self.key_prefix = f"{self.index_name}:"
         self._index_dim: int | None = None

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -113,6 +113,58 @@ def _build_reasoning_item(
     }
 
 
+def _reasoning_item_from_output_item(item: object) -> _BuiltReasoningItem | None:
+    from openai.types.responses import ResponseReasoningItem
+
+    if isinstance(item, ResponseReasoningItem):
+        return _build_reasoning_item(
+            item_id=item.id,
+            encrypted_content=getattr(item, "encrypted_content", None),
+            summary_raw=item.summary,
+        )
+    if isinstance(item, dict) and item.get("type") == "reasoning":
+        return _build_reasoning_item(
+            item_id=item.get("id", ""),
+            encrypted_content=item.get("encrypted_content"),
+            summary_raw=item.get("summary"),
+        )
+    return None
+
+
+def _reasoning_items_from_output_items(output_items: Sequence[object]) -> tuple[_BuiltReasoningItem, ...]:
+    return tuple(
+        reasoning_item
+        for reasoning_item in (_reasoning_item_from_output_item(item) for item in output_items)
+        if reasoning_item is not None
+    )
+
+
+def _as_chat_reasoning_items(
+    reasoning_items: Sequence[_BuiltReasoningItem],
+) -> list[ChatCompletionReasoningItem] | None:
+    if not reasoning_items:
+        return None
+    # cast-ok: _BuiltReasoningItem is the structural shape ChatCompletionReasoningItem
+    # describes, and TypedDict invariance is what stops the two from unifying here.
+    return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
+
+
+def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Literal["length", "content_filter"]:
+    if incomplete_reason == "content_filter":
+        return "content_filter"
+    return "length"
+
+
+def _incomplete_reason_from_response_payload(response_payload: object) -> str | None:
+    if not isinstance(response_payload, Mapping):
+        return None
+    incomplete_details: Final = response_payload.get("incomplete_details")
+    if not isinstance(incomplete_details, Mapping):
+        return None
+    reason: Final = incomplete_details.get("reason")
+    return reason if isinstance(reason, str) else None
+
+
 class _ChatToolCallDict(ChatCompletionToolCallChunk, total=False):
     provider_specific_fields: Mapping[str, object]
 
@@ -657,6 +709,27 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         return choices
 
+    @staticmethod
+    def _build_empty_incomplete_choice(
+        output_items: Sequence[object],
+        finish_reason: Literal["length", "content_filter"],
+    ) -> "Choices":
+        from litellm.types.utils import Choices, Message
+
+        reasoning_items: Final = _reasoning_items_from_output_items(output_items)
+        reasoning_content: Final = " ".join(
+            summary_block["text"]
+            for reasoning_item in reasoning_items
+            for summary_block in reasoning_item["summary"]
+            if summary_block.get("text")
+        )
+        message: Final = Message(
+            content="",
+            reasoning_content=reasoning_content if reasoning_content else None,
+            reasoning_items=_as_chat_reasoning_items(reasoning_items),
+        )
+        return Choices(message=message, finish_reason=finish_reason, index=0)
+
     @classmethod
     def _extract_output_from_completed_event(cls, parsed_chunk: Mapping[str, object]) -> list[dict[str, object]] | None:
         response_payload: Final = parsed_chunk.get("response")
@@ -763,11 +836,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             handle_raw_dict_callback=self._handle_raw_dict_response_item,
         )
 
-        if len(choices) == 0:
-            if raw_response.incomplete_details is not None and raw_response.incomplete_details.reason is not None:
-                raise ValueError(f"{model} unable to complete request: {raw_response.incomplete_details.reason}")
+        response_is_incomplete: Final = raw_response.status == "incomplete" or (
+            raw_response.incomplete_details is not None and raw_response.incomplete_details.reason is not None
+        )
+
+        if len(choices) == 0 and not response_is_incomplete:
+            raise ValueError(f"Unknown items in responses API response: {output_items}")
+
+        if response_is_incomplete:
+            incomplete_finish_reason: Final = _map_incomplete_reason_to_finish_reason(
+                raw_response.incomplete_details.reason if raw_response.incomplete_details is not None else None
+            )
+            if len(choices) == 0:
+                choices.append(self._build_empty_incomplete_choice(output_items, incomplete_finish_reason))
             else:
-                raise ValueError(f"Unknown items in responses API response: {output_items}")
+                for choice in choices:
+                    choice.finish_reason = incomplete_finish_reason
 
         setattr(model_response, "choices", choices)
 
@@ -1392,12 +1476,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                         )
                     ]
                 )
-        elif event_type == "response.completed":
-            # Response is fully complete - now we can signal is_finished=True
-            # This ensures we don't prematurely end the stream before tool_calls arrive
-
-            # Check if response contains function_call items in output
-            # to determine correct finish_reason
+        elif event_type in ("response.completed", "response.incomplete"):
             response_data: Final = parsed_chunk.get("response", {})
             output_items: Final = response_data.get("output", []) if response_data else []
 
@@ -1407,25 +1486,14 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                 if isinstance(item, dict)
             )
 
-            finish_reason: Final = "tool_calls" if has_function_calls else "stop"
+            finish_reason: Final = (
+                _map_incomplete_reason_to_finish_reason(_incomplete_reason_from_response_payload(response_data))
+                if event_type == "response.incomplete"
+                else ("tool_calls" if has_function_calls else "stop")
+            )
 
-            # Extract reasoning items with encrypted_content for round-tripping
-            completed_reasoning_items: list[_BuiltReasoningItem] | None = None
-            for item in output_items:
-                if not isinstance(item, dict) or item.get("type") != "reasoning":
-                    continue
-                if completed_reasoning_items is None:
-                    completed_reasoning_items = []
-                completed_reasoning_items.append(
-                    _build_reasoning_item(
-                        item_id=item.get("id", ""),
-                        encrypted_content=item.get("encrypted_content"),
-                        summary_raw=item.get("summary"),
-                    )
-                )
-            completed_reasoning_items_typed: Final = cast(
-                list[ChatCompletionReasoningItem] | None,
-                completed_reasoning_items,
+            terminal_reasoning_items_typed: Final = _as_chat_reasoning_items(
+                _reasoning_items_from_output_items(output_items)
             )
 
             usage = None
@@ -1439,7 +1507,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                         index=0,
                         delta=Delta(
                             content="",
-                            reasoning_items=completed_reasoning_items_typed,
+                            reasoning_items=terminal_reasoning_items_typed,
                         ),
                         finish_reason=finish_reason,
                     )

litellm/constants.py

@@ -436,6 +436,9 @@ DEFAULT_REQUEST_TIMEOUT_SECONDS: Final[float] = 6000.0
 # deadline and connect handshake (see ``http_handler`` cached handler paths).
 COMPLETION_HTTP_FALLBACK_SECONDS: Final[float] = 600.0
 HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: Final[float] = 5.0
+SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS: Final[float] = float(
+    os.getenv("SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS", "5.0")
+)
 request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS))))
 request_timeout_explicitly_set: bool = "REQUEST_TIMEOUT" in os.environ
 DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000))  # 10 minutes
@@ -779,6 +782,7 @@ openai_compatible_endpoints: Final[list] = [
     "https://api.libertai.io/v1",
     "https://pinstripes.io/v1",
     "https://api.meta.ai/v1",
+    "https://api.cognition.ai/v1",
 ]
 
 
@@ -846,6 +850,7 @@ openai_compatible_providers: Final[list] = [
     "pinstripes",  # Pinstripes - JSON-configured provider
     "darkbloom",
     "meta",  # Meta Model API (Muse Spark) - JSON-configured provider
+    "cognition",
 ]
 openai_text_completion_compatible_providers: Final[list] = [  # providers that support `/v1/completions`
     "together_ai",
@@ -1537,6 +1542,11 @@ DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_
 PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597))
 RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_BATCH_SIZE", "500")))
 RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN", "100")))
+RESET_BUDGET_JOB_NAME: Final = "reset_budget_job"
+# Comfortably longer than one PROXY_BUDGET_RESCHEDULER_MIN_TIME tick, so a healthy
+# leader keeps the lease across its own run, and a crashed one strands the sweep for
+# at most a single tick.
+RESET_BUDGET_JOB_LOCK_TTL_SECONDS: Final[int] = 900
 PROXY_BATCH_POLLING_INTERVAL: Final = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 3600))
 MAX_OBJECTS_PER_POLL_CYCLE: Final = max(1, int(os.getenv("MAX_OBJECTS_PER_POLL_CYCLE", 50)))
 MANAGED_OBJECT_STALENESS_CUTOFF_DAYS: Final = max(1, int(os.getenv("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7)))
@@ -1601,6 +1611,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     "public_model_groups_links",
     "cost_discount_config",
     "cost_margin_config",
+    "block_requests_for_models_without_pricing",
     "budget_exceeded_throttle_percentage",
     # Every field editable from the Admin UI (proxy_server._GENERAL_SETTINGS_UI_LITELLM_FIELDS)
     # must be listed here so a DB write from one worker overrides the live litellm attribute on

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -349,6 +349,9 @@ def get_llm_provider(
                     elif endpoint == "https://api.meta.ai/v1":
                         custom_llm_provider = "meta"
                         dynamic_api_key = get_secret_str("META_API_KEY")
+                    elif (json_provider := JSONProviderRegistry.get_by_base_url(endpoint)) is not None:
+                        custom_llm_provider = json_provider.slug
+                        dynamic_api_key = api_key if api_key is not None else get_secret_str(json_provider.api_key_env)
 
                     if api_base is not None and not isinstance(api_base, str):
                         raise Exception(f"api base needs to be a string. api_base={api_base}")

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1371,6 +1371,7 @@ class CostCalculatorUtils:
             return fal_ai_image_cost_calculator(
                 model=model,
                 image_response=completion_response,
+                optional_params=optional_params,
             )
         elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
             from litellm.llms.runwayml.cost_calculator import (

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1200,13 +1200,14 @@ def _encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: st
     return tool_call_id
 
 
-def _get_thought_signature_from_tool(tool: dict, model: str | None = None) -> str | None:
+def _get_thought_signature_from_tool(tool: dict) -> str | None:
     """Extract thought signature from tool call's provider_specific_fields.
 
     If not provided try to extract thought signature from tool call id
 
     Checks both tool.provider_specific_fields and tool.function.provider_specific_fields.
-    If no signature is found and model is gemini-3, returns a dummy signature.
+    Returns None when the tool call carries no signature; callers decide whether a
+    placeholder signature is needed.
     """
     # First check tool's provider_specific_fields
     provider_fields: Final = tool.get("provider_specific_fields") or {}
@@ -1236,13 +1237,6 @@ def _get_thought_signature_from_tool(tool: dict, model: str | None = None) -> st
         if len(parts) == 2:
             _, signature = parts
             return signature
-    # If no signature found and model is gemini-3, return dummy signature
-    from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
-        VertexGeminiConfig,
-    )
-
-    if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
-        return _get_dummy_thought_signature()
     return None
 
 
@@ -1251,10 +1245,14 @@ def _get_dummy_thought_signature() -> str:
 
     This is used when transferring conversation history from older models
     (like gemini-2.5-flash) to gemini-3, which requires thought_signature
-    for strict validation.
+    for strict validation. Google documents it as a last resort that "will
+    negatively impact model performance", so callers must only fall back to it
+    when no real signature is available.
+
+    See:
+    https://ai.google.dev/gemini-api/docs/thought-signatures#faqs
+    https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures
     """
-    # Return a base64-encoded dummy signature string
-    # Below dummy signature is recommended by google - https://ai.google.dev/gemini-api/docs/thought-signatures#faqs
     dummy_data: Final = b"skip_thought_signature_validator"
     return base64.b64encode(dummy_data).decode("utf-8")
 
@@ -1312,8 +1310,10 @@ def convert_to_gemini_tool_call_invoke(
             VertexGeminiConfig,
         )
 
+        needs_dummy_signature: Final = model is not None and VertexGeminiConfig._is_gemini_3_or_newer(model)
+
         if tool_calls is not None:
-            for idx, tool in enumerate(tool_calls):
+            for tool in tool_calls:
                 if "function" in tool:
                     gemini_function_call: VertexFunctionCall | None = _gemini_tool_call_invoke_helper(
                         function_call_params=tool["function"],
@@ -1321,7 +1321,13 @@ def convert_to_gemini_tool_call_invoke(
                     )
                     if gemini_function_call is not None:
                         part_dict: VertexPartType = {"function_call": gemini_function_call}
-                        thought_signature = _get_thought_signature_from_tool(dict(tool), model=model)
+                        thought_signature = _get_thought_signature_from_tool(dict(tool))
+                        # Gemini signs only the first functionCall part of a parallel batch, so scope the
+                        # placeholder fallback to that part instead of fabricating one per sibling call:
+                        # https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/thinking/thought-signatures#parallel_function_calling_example
+                        is_first_function_call = len(_parts_list) == 0
+                        if not thought_signature and is_first_function_call and needs_dummy_signature:
+                            thought_signature = _get_dummy_thought_signature()
                         if thought_signature:
                             part_dict["thoughtSignature"] = thought_signature
 
@@ -1344,7 +1350,7 @@ def convert_to_gemini_tool_call_invoke(
                     thought_signature = provider_fields.get("thought_signature")
 
                 # If no signature found and model is gemini-3, use dummy signature
-                if not thought_signature and model and VertexGeminiConfig._is_gemini_3_or_newer(model):
+                if not thought_signature and needs_dummy_signature:
                     thought_signature = _get_dummy_thought_signature()
 
                 if thought_signature:

litellm/litellm_core_utils/streaming_handler.py

@@ -191,7 +191,7 @@ class CustomStreamWrapper:
         custom_llm_provider: str | None = None,
         stream_options=None,
         make_call: Callable | None = None,
-        _response_headers: dict | None = None,
+        _response_headers: dict | httpx.Headers | None = None,
     ):
         self.model = model
         self.make_call = make_call
@@ -2315,10 +2315,18 @@ class CustomStreamWrapper:
         if self.logging_obj is None or not self.chunks:
             return
         try:
-            partial_response: Final = litellm.stream_chunk_builder(chunks=self.chunks)
+            partial_response: Final = litellm.stream_chunk_builder(
+                chunks=self.chunks,
+                messages=self.messages if isinstance(self.messages, list) else None,
+            )
+            if partial_response is None:
+                return
             usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
             if usage is None:
                 return
+            if self.model:
+                partial_response.model = self.model
+            backfill_missing_cache_usage_fields(usage)
             self.logging_obj.model_call_details["combined_usage_object"] = usage
             self.logging_obj.model_call_details["response_cost"] = (
                 self.logging_obj._response_cost_calculator(result=partial_response) or 0.0
@@ -2439,6 +2447,35 @@ class CustomStreamWrapper:
         return chunk
 
 
+def _cache_token_count(details: PromptTokensDetailsWrapper | None, keys: tuple[str, ...]) -> int:
+    for key in keys:
+        value = getattr(details, key, None)
+        if isinstance(value, int) and not isinstance(value, bool) and value:
+            return value
+    return 0
+
+
+def backfill_missing_cache_usage_fields(usage: Usage) -> None:
+    """Give partial-stream usage the same cache fields a complete stream reports.
+
+    Carries OpenAI-style ``prompt_tokens_details`` counts up to the Anthropic-style
+    top-level keys, defaulting to zero. It must carry the real count rather than a
+    flat zero: downstream readers treat these keys as authoritative once present and
+    skip their own normalization, so a zero here would overwrite a real cache read.
+    """
+    details: Final = usage.prompt_tokens_details
+    if getattr(usage, "cache_read_input_tokens", None) is None:
+        usage.cache_read_input_tokens = _cache_token_count(  # rebind-ok: in-place backfill is the contract
+            details, ("cached_tokens",)
+        )
+    if getattr(usage, "cache_creation_input_tokens", None) is None:
+        usage.cache_creation_input_tokens = _cache_token_count(  # rebind-ok: in-place backfill is the contract
+            details, ("cache_write_tokens", "cache_creation_tokens")
+        )
+    if usage.prompt_tokens_details is None:
+        usage.prompt_tokens_details = PromptTokensDetailsWrapper(cached_tokens=0)  # rebind-ok: backfill in place
+
+
 _TokenDetails = TypeVar("_TokenDetails", PromptTokensDetailsWrapper, CompletionTokensDetailsWrapper)
 
 

litellm/llms/anthropic/chat/transformation.py

@@ -2216,7 +2216,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
     def calculate_usage(
         self,
-        usage_object: dict,
+        usage_object: Mapping[str, Any],
         reasoning_content: str | None,
         completion_response: dict | None = None,
         speed: str | None = None,

litellm/llms/bedrock/chat/converse_handler.py

@@ -35,7 +35,7 @@ def make_sync_call(
     json_mode: bool | None = False,
     fake_stream: bool = False,
     stream_chunk_size: int | None = None,
-):
+) -> tuple[Any, httpx.Headers]:
     if client is None:
         client = _get_httpx_client()  # Create a new client if none provided
 
@@ -76,7 +76,7 @@ def make_sync_call(
         additional_args={"complete_input_dict": data},
     )
 
-    return completion_stream
+    return completion_stream, response.headers
 
 
 class BedrockConverseLLM(BaseAWSLLM):
@@ -134,7 +134,7 @@ class BedrockConverseLLM(BaseAWSLLM):
             },
         )
 
-        completion_stream: Final = await make_call(
+        completion_stream, response_headers = await make_call(
             client=client,
             api_base=api_base,
             headers=dict(prepped.headers),
@@ -151,6 +151,7 @@ class BedrockConverseLLM(BaseAWSLLM):
             model=model,
             custom_llm_provider="bedrock",
             logging_obj=logging_obj,
+            _response_headers=response_headers,
         )
         return streaming_response
 
@@ -232,7 +233,7 @@ class BedrockConverseLLM(BaseAWSLLM):
         except httpx.TimeoutException:
             raise BedrockError(status_code=408, message="Timeout error occurred.")
 
-        return litellm.AmazonConverseConfig()._transform_response(
+        transformed_response: Final = litellm.AmazonConverseConfig()._transform_response(
             model=model,
             response=response,
             model_response=model_response,
@@ -244,6 +245,8 @@ class BedrockConverseLLM(BaseAWSLLM):
             optional_params=optional_params,
             encoding=encoding,
         )
+        transformed_response.set_provider_response_headers(response.headers)
+        return transformed_response
 
     def completion(
         self,
@@ -541,7 +544,7 @@ class BedrockConverseLLM(BaseAWSLLM):
             client = client
 
         if stream is not None and stream is True:
-            completion_stream: Final = make_sync_call(
+            completion_stream, response_headers = make_sync_call(
                 client=(client if client is not None and isinstance(client, HTTPHandler) else None),
                 api_base=proxy_endpoint_url,
                 headers=prepped.headers,
@@ -558,6 +561,7 @@ class BedrockConverseLLM(BaseAWSLLM):
                 model=model,
                 custom_llm_provider="bedrock",
                 logging_obj=logging_obj,
+                _response_headers=response_headers,
             )
 
             return streaming_response
@@ -578,7 +582,7 @@ class BedrockConverseLLM(BaseAWSLLM):
         except httpx.TimeoutException:
             raise BedrockError(status_code=408, message="Timeout error occurred.")
 
-        return litellm.AmazonConverseConfig()._transform_response(
+        sync_transformed_response: Final = litellm.AmazonConverseConfig()._transform_response(
             model=model,
             response=response,
             model_response=model_response,
@@ -590,3 +594,5 @@ class BedrockConverseLLM(BaseAWSLLM):
             optional_params=optional_params,
             encoding=encoding,
         )
+        sync_transformed_response.set_provider_response_headers(response.headers)
+        return sync_transformed_response

litellm/llms/bedrock/chat/invoke_handler.py

@@ -163,7 +163,7 @@ async def make_call(
     json_mode: bool | None = False,
     bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
     stream_chunk_size: int | None = None,
-):
+) -> tuple[Any, httpx.Headers]:
     try:
         if client is None:
             client = get_async_httpx_client(
@@ -225,7 +225,7 @@ async def make_call(
             additional_args={"complete_input_dict": data},
         )
 
-        return completion_stream
+        return completion_stream, response.headers
     except httpx.HTTPStatusError as err:
         error_code: Final = err.response.status_code
         raise BedrockError(status_code=error_code, message=err.response.text)
@@ -248,7 +248,7 @@ def make_sync_call(
     json_mode: bool | None = False,
     bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
     stream_chunk_size: int | None = None,
-):
+) -> tuple[Any, httpx.Headers]:
     try:
         if client is None:
             client = _get_httpx_client(
@@ -309,7 +309,7 @@ def make_sync_call(
             additional_args={"complete_input_dict": data},
         )
 
-        return completion_stream
+        return completion_stream, response.headers
     except httpx.HTTPStatusError as err:
         error_code: Final = err.response.status_code
         raise BedrockError(status_code=error_code, message=err.response.text)

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -1,7 +1,6 @@
 import copy
 import json
 import time
-from functools import partial
 from typing import TYPE_CHECKING, Any, Final, cast, get_args
 
 import httpx
@@ -446,24 +445,24 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
     ) -> CustomStreamWrapper:
+        completion_stream, response_headers = await make_call(
+            client=client,
+            api_base=api_base,
+            headers=headers,
+            data=json.dumps(data),
+            model=model,
+            messages=messages,
+            logging_obj=logging_obj,
+            fake_stream=True if "ai21" in api_base else False,
+            bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
+            json_mode=json_mode,
+        )
         streaming_response: Final = CustomStreamWrapper(
-            completion_stream=None,
-            make_call=partial(
-                make_call,
-                client=client,
-                api_base=api_base,
-                headers=headers,
-                data=json.dumps(data),
-                model=model,
-                messages=messages,
-                logging_obj=logging_obj,
-                fake_stream=True if "ai21" in api_base else False,
-                bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
-                json_mode=json_mode,
-            ),
+            completion_stream=completion_stream,
             model=model,
             custom_llm_provider="bedrock",
             logging_obj=logging_obj,
+            _response_headers=response_headers,
         )
         return streaming_response
 
@@ -481,27 +480,28 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
     ) -> CustomStreamWrapper:
-        if client is None or isinstance(client, AsyncHTTPHandler):
-            client = _get_httpx_client(params={})
+        sync_client: Final = (
+            _get_httpx_client(params={}) if client is None or isinstance(client, AsyncHTTPHandler) else client
+        )
+        completion_stream, response_headers = make_sync_call(
+            client=sync_client,
+            api_base=api_base,
+            headers=headers,
+            data=json.dumps(data),
+            signed_json_body=signed_json_body,
+            model=model,
+            messages=messages,
+            logging_obj=logging_obj,
+            fake_stream=True if "ai21" in api_base else False,
+            bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
+            json_mode=json_mode,
+        )
         streaming_response: Final = CustomStreamWrapper(
-            completion_stream=None,
-            make_call=partial(
-                make_sync_call,
-                client=client,
-                api_base=api_base,
-                headers=headers,
-                data=json.dumps(data),
-                signed_json_body=signed_json_body,
-                model=model,
-                messages=messages,
-                logging_obj=logging_obj,
-                fake_stream=True if "ai21" in api_base else False,
-                bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
-                json_mode=json_mode,
-            ),
+            completion_stream=completion_stream,
             model=model,
             custom_llm_provider="bedrock",
             logging_obj=logging_obj,
+            _response_headers=response_headers,
         )
         return streaming_response
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -635,6 +635,7 @@ class BaseLLMHTTPHandler:
                 model=model,
                 custom_llm_provider=custom_llm_provider,
                 logging_obj=logging_obj,
+                _response_headers=headers,
             )
 
         if client is None or not isinstance(client, HTTPHandler):
@@ -798,6 +799,7 @@ class BaseLLMHTTPHandler:
             model=model,
             custom_llm_provider=custom_llm_provider,
             logging_obj=logging_obj,
+            _response_headers=_response_headers,
         )
         return streamwrapper
 

litellm/llms/fal_ai/cost_calculator.py

@@ -1,25 +1,75 @@
-from typing import Any, Final
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
 
 import litellm
 from litellm.types.utils import ImageResponse
 
+FAL_KEYED_PRICING_DEFAULT_QUALITY: Final[str] = "high"
+FAL_TEXT_TO_IMAGE_DEFAULT_SIZE: Final[str] = "1024-x-768"
+FAL_NAMED_IMAGE_SIZES: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "square_hd": "1024-x-1024",
+        "square": "512-x-512",
+        "portrait_4_3": "768-x-1024",
+        "portrait_16_9": "576-x-1024",
+        "landscape_4_3": "1024-x-768",
+        "landscape_16_9": "1024-x-576",
+    }
+)
+
+
+def _keyed_size(model: str, optional_params: Mapping[str, object]) -> str | None:
+    image_size: Final = optional_params.get("image_size")
+    if image_size is None:
+        return None if model.endswith("/edit") else FAL_TEXT_TO_IMAGE_DEFAULT_SIZE
+    if isinstance(image_size, Mapping):
+        width: Final = image_size.get("width")
+        height: Final = image_size.get("height")
+        if isinstance(width, int) and isinstance(height, int):
+            return f"{width}-x-{height}"
+        return None
+    if isinstance(image_size, str):
+        return FAL_NAMED_IMAGE_SIZES.get(image_size)
+    return None
+
+
+def _keyed_cost_per_image(model: str, optional_params: Mapping[str, object] | None) -> float | None:
+    if optional_params is None:
+        return None
+    size: Final = _keyed_size(model=model, optional_params=optional_params)
+    if size is None:
+        return None
+    raw_quality: Final = optional_params.get("quality")
+    quality: Final = (
+        raw_quality if isinstance(raw_quality, str) and raw_quality != "auto" else FAL_KEYED_PRICING_DEFAULT_QUALITY
+    )
+    keyed_entry: Final = litellm.model_cost.get(f"fal_ai/{quality}/{size}/{model}")
+    if keyed_entry is None:
+        return None
+    keyed_cost: Final = keyed_entry.get("output_cost_per_image")
+    return float(keyed_cost) if isinstance(keyed_cost, (int, float)) else None
+
 
 def cost_calculator(
     model: str,
-    image_response: Any,
+    image_response: object,
+    optional_params: Mapping[str, object] | None = None,
 ) -> float:
     """
     fal.ai image generation cost calculator
     """
+    if not isinstance(image_response, ImageResponse):
+        raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
+    # the proxy cost path passes the provider-prefixed model name
+    model = model.removeprefix(f"{litellm.LlmProviders.FAL_AI.value}/")
+    num_images: Final[int] = len(image_response.data) if image_response.data else 0
+    keyed_cost_per_image: Final = _keyed_cost_per_image(model=model, optional_params=optional_params)
+    if keyed_cost_per_image is not None:
+        return keyed_cost_per_image * num_images
     _model_info: Final = litellm.get_model_info(
         model=model,
         custom_llm_provider=litellm.LlmProviders.FAL_AI.value,
     )
     output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0
-    num_images: int = 0
-    if isinstance(image_response, ImageResponse):
-        if image_response.data:
-            num_images = len(image_response.data)
-        return output_cost_per_image * num_images
-    else:
-        raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
+    return output_cost_per_image * num_images

litellm/llms/openai_like/json_loader.py

@@ -65,6 +65,11 @@ class JSONProviderRegistry:
         """Check if a provider is defined via JSON"""
         return slug in cls._providers
 
+    @classmethod
+    def get_by_base_url(cls, base_url: str) -> SimpleProviderConfig | None:
+        """Get a provider configuration by its default base url"""
+        return next((provider for provider in cls._providers.values() if provider.base_url == base_url), None)
+
     @classmethod
     def supports_responses_api(cls, slug: str) -> bool:
         """Check if a JSON provider supports the Responses API"""

litellm/llms/openai_like/providers.json

@@ -175,6 +175,11 @@
     "base_class": "openai_gpt",
     "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
   },
+  "cognition": {
+    "base_url": "https://api.cognition.ai/v1",
+    "api_key_env": "COGNITION_API_KEY",
+    "api_base_env": "COGNITION_API_BASE"
+  },
   "pinstripes": {
     "base_url": "https://pinstripes.io/v1",
     "api_key_env": "PINSTRIPES_API_KEY",

litellm/llms/sagemaker/chat/transformation.py

@@ -54,7 +54,30 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
         api_key: str | None = None,
         api_base: str | None = None,
     ) -> dict:
-        return headers
+        inference_component_name: Final = optional_params.get("model_id")
+        if not isinstance(inference_component_name, str):
+            return headers
+        return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name}
+
+    def transform_request(
+        self,
+        model: str,
+        messages: list[AllMessageValues],  # mutable-ok: matches the base chat transform signature
+        optional_params: dict,  # mutable-ok: matches the base chat transform signature
+        litellm_params: dict,  # mutable-ok: matches the base chat transform signature
+        headers: dict,  # mutable-ok: matches the base chat transform signature
+    ) -> dict:  # mutable-ok: the handler sends this body straight to httpx
+        request: Final = super().transform_request(
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+        served_model_name: Final = litellm_params.get("hf_model_name")
+        if not isinstance(served_model_name, str):
+            return request
+        return {**request, "model": served_model_name}
 
     def get_complete_url(
         self,

litellm/llms/vertex_ai/gemini/transformation.py

@@ -645,10 +645,9 @@ def _collect_tool_call_thought_signatures(
     the text part as well would send two copies and double-bill the previous
     turn's reasoning tokens on gemini-3 and newer models.
 
-    Detection deliberately calls _get_thought_signature_from_tool without the
-    model argument: with a gemini-3 model that helper synthesizes a dummy
-    signature for unsigned tool calls, which must not suppress a real
-    text-part signature (e.g. replaying gemini-2.5 history to a newer model).
+    Only real signatures count here; a synthesized placeholder must not
+    suppress a genuine text-part signature (e.g. replaying gemini-2.5 history
+    to a newer model).
     """
     signatures: tuple[str, ...] = ()
 

litellm/main.py

@@ -5974,7 +5974,7 @@ def embedding(
     # Optional params
     dimensions: int | None = None,
     encoding_format: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float = 600,  # default to 10 minutes
     # set api_base, api_version, api_key
     api_base: str | None = None,
     api_version: str | None = None,
@@ -6000,7 +6000,7 @@ def embedding(
     # Optional params
     dimensions: int | None = None,
     encoding_format: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float = 600,  # default to 10 minutes
     # set api_base, api_version, api_key
     api_base: str | None = None,
     api_version: str | None = None,
@@ -6027,7 +6027,7 @@ def embedding(
     # Optional params
     dimensions: int | None = None,
     encoding_format: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float = 600,  # default to 10 minutes
     # set api_base, api_version, api_key
     api_base: str | None = None,
     api_version: str | None = None,

litellm/provider_endpoints_support_backup.json

@@ -528,6 +528,23 @@
         "interactions": true
       }
     },
+    "cognition": {
+      "display_name": "Cognition (`cognition`)",
+      "url": "https://docs.litellm.ai/docs/providers/cognition",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": true,
+        "responses": true,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false
+      }
+    },
     "cohere": {
       "display_name": "Cohere (`cohere`)",
       "url": "https://docs.litellm.ai/docs/providers/cohere",

litellm/proxy/_types.py

@@ -3746,6 +3746,8 @@ class ProxyErrorTypes(str, enum.Enum):
     Project does not have access to the model
     """
 
+    model_cost_map_missing = "model_cost_map_missing"
+
     expired_key = "expired_key"
     """
     Key has expired

litellm/proxy/auth/auth_checks.py

@@ -456,6 +456,103 @@ def _is_cost_explicitly_configured(model: str, llm_router: "Router") -> bool:
     return False
 
 
+_EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({})
+
+
+def _is_positive_cost(value: object) -> bool:
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0
+
+
+def _entry_has_priced_metric(entry: Mapping[str, object]) -> bool:
+    if entry.get("tiered_pricing") is not None:
+        return True
+    for key, value in entry.items():
+        if "cost_per" not in key:
+            continue
+        if _is_positive_cost(value):
+            return True
+        if isinstance(value, dict) and any(_is_positive_cost(nested) for nested in value.values()):
+            return True
+    return False
+
+
+def _entry_declares_price(entry: Mapping[str, object]) -> bool:
+    return any("cost_per" in key or key == "tiered_pricing" for key in entry)
+
+
+def _model_group_has_pricing(model: str, llm_router: "Router") -> bool:
+    """
+    A model group counts as priced when a deployment overrides any *cost_per* field or
+    tiered_pricing in its litellm_params, even at zero, or when its resolved model info carries
+    tiered_pricing or a positive price on any billed metric (tokens, characters, seconds, pages,
+    images, queries, ...), so models billed by a non-token metric are not treated as unpriced.
+    """
+    for deployment in llm_router.get_model_list(model_name=model) or ():
+        litellm_params = deployment.get("litellm_params") or _EMPTY_COST_ENTRY
+        if _entry_declares_price(litellm_params):
+            return True
+
+        model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
+        if model_id is None:
+            continue
+
+        model_info = llm_router.get_deployment_model_info(
+            model_id=model_id, model_name=litellm_params.get("model") or ""
+        )
+        if model_info is not None and _entry_has_priced_metric(model_info):
+            return True
+
+    return False
+
+
+def _group_declares_explicit_cost(model: str, llm_router: "Router") -> bool:
+    """
+    Alias-aware counterpart to ``_is_cost_explicitly_configured``, which resolves the model group
+    the same way ``_model_group_has_pricing`` does. A deployment that prices itself through its
+    ``model_info`` block lands in the cost map under its deployment id rather than in its
+    litellm_params, and reaching that entry through the router's own resolution keeps an alias
+    pointing at such a group from being read as unpriced.
+    """
+    for deployment in llm_router.get_model_list(model_name=model) or ():
+        model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
+        if model_id is None:
+            continue
+        raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
+        if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry:
+            return True
+    return False
+
+
+def model_has_no_cost_mapping(model: str | None, llm_router: Router | None) -> bool:
+    if not model or llm_router is None:
+        return False
+
+    if llm_router.get_model_group_info(model_group=model) is None:
+        return False
+
+    if _model_group_has_pricing(model=model, llm_router=llm_router):
+        return False
+
+    return not _group_declares_explicit_cost(model=model, llm_router=llm_router)
+
+
+def _unpriced_models_in_request(model: str | list[str] | None, llm_router: Router | None) -> tuple[str, ...]:
+    candidates: Final = (model,) if isinstance(model, str) else tuple(model or ())
+    return tuple(
+        candidate for candidate in candidates if model_has_no_cost_mapping(model=candidate, llm_router=llm_router)
+    )
+
+
+def _unpriced_models_block_message(models: tuple[str, ...]) -> str:
+    names: Final = ", ".join(f"'{model}'" for model in models)
+    subject: Final = f"Model {names} has" if len(models) == 1 else f"Models {names} have"
+    return (
+        f"{subject} no pricing in the cost map, so litellm cannot price the request. "
+        "Requests for unpriced models are blocked because 'block_requests_for_models_without_pricing' "
+        "is enabled. Add pricing (input_cost_per_token/output_cost_per_token) to allow the request."
+    )
+
+
 async def _run_project_checks(
     project_object: LiteLLM_ProjectTableCachedObj | None,
     _model: str | list[str] | None,
@@ -726,6 +823,19 @@ async def common_checks(
         and (route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route))
     )
 
+    unpriced_models: Final = (
+        _unpriced_models_in_request(model=_model, llm_router=llm_router)
+        if litellm.block_requests_for_models_without_pricing and RouteChecks.is_llm_api_route(route=route)
+        else ()
+    )
+    if unpriced_models:
+        raise ProxyException(
+            message=_unpriced_models_block_message(unpriced_models),
+            type=ProxyErrorTypes.model_cost_map_missing,
+            param="model",
+            code=status.HTTP_403_FORBIDDEN,
+        )
+
     # 1. If team is blocked
     if team_object is not None and team_object.blocked is True:
         raise Exception(f"Team={team_object.team_id} is blocked. Update via `/team/unblock` if you're an admin.")

litellm/proxy/common_request_processing.py

@@ -43,6 +43,9 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import (
     get_response_headers,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.litellm_core_utils.streaming_handler import (
+    backfill_missing_cache_usage_fields,
+)
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
 from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
 from litellm.proxy.auth.auth_utils import check_response_size_is_safe
@@ -158,7 +161,7 @@ ProxyRouteType: TypeAlias = Literal[
     "acancel_run",
     "adelete_run",
 ]
-from litellm.types.utils import ServerToolUse
+from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 
 # Type alias for streaming chunk serializer (chunk after hooks + cost injection -> wire format)
 StreamChunkSerializer = Callable[[Any], str]
@@ -274,6 +277,43 @@ def _deferred_stream_logging_is_armed(request_data: dict) -> bool:
     )
 
 
+def _assembled_model_came_from_a_later_chunk(chunks: list, assembled_model: object) -> bool:
+    """Report whether stream_chunk_builder picked a model the first chunk did not carry.
+
+    Azure Model Router puts the routed model on the chunks after the first one, and the
+    proxy deliberately leaves those chunks unrestamped so the builder can recover it.
+
+    A stored chunk that carries usage is a pre-restamp copy of the one the proxy saw, so
+    an alias-restamped stream reaches the builder with the same shape: a first chunk that
+    disagrees with the rest. Those two are only told apart by what the client asked for.
+    """
+    first_chunk: Final = chunks[0]
+    first_chunk_model: Final = (
+        first_chunk.get("model") if isinstance(first_chunk, dict) else getattr(first_chunk, "model", None)
+    )
+    return (
+        isinstance(first_chunk_model, str)
+        and isinstance(assembled_model, str)
+        and bool(assembled_model)
+        and assembled_model != first_chunk_model
+    )
+
+
+def _assembled_model_is_the_name_the_client_asked_for(request_data: dict, assembled_model: object) -> bool:
+    """Report whether the assembled model is the public name the proxy stamps onto chunks.
+
+    That stamp is what leaves an unpriced alias on the partial response, so the deployment's
+    own model has to go back on before the row is costed. Pre-call processing rewrites
+    `request_data["model"]` for aliasing and routing, so the client's own name wins when it
+    is there, in the same order the proxy picks the name it stamps.
+    """
+    client_requested_model: Final = request_data.get("_litellm_client_requested_model")
+    stamped_model: Final = (
+        client_requested_model if isinstance(client_requested_model, str) else request_data.get("model")
+    )
+    return isinstance(stamped_model, str) and assembled_model == stamped_model
+
+
 async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, response: object) -> bool:
     """
     A client disconnect throws GeneratorExit/CancelledError into the streaming
@@ -324,6 +364,15 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
         return False
     if partial_response is None:
         return False
+    wrapper_model: Final = getattr(response, "model", None)
+    builder_recovered_the_routed_model: Final = _assembled_model_came_from_a_later_chunk(
+        chunks, partial_response.model
+    ) and not _assembled_model_is_the_name_the_client_asked_for(request_data, partial_response.model)
+    if isinstance(wrapper_model, str) and wrapper_model and not builder_recovered_the_routed_model:
+        partial_response.model = wrapper_model
+    partial_usage: Final = getattr(partial_response, "usage", None)
+    if isinstance(partial_usage, Usage):
+        backfill_missing_cache_usage_fields(partial_usage)
     try:
         await logging_obj.dispatch_success_handlers(
             partial_response,
@@ -3321,7 +3370,9 @@ class ProxyBaseLLMRequestProcessing:
                         str_so_far += str(chunk.get("content", ""))
 
                     model_name = request_data.get("model", "")
-                    chunk = ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection(chunk, model_name)
+                    chunk = ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection(
+                        chunk, model_name, request_data.get("litellm_logging_obj")
+                    )
 
                 # Set before the yield: an async generator suspends at the yield,
                 # so a GeneratorExit on client disconnect is raised there and any
@@ -3418,20 +3469,27 @@ class ProxyBaseLLMRequestProcessing:
 
     @overload
     @staticmethod
-    def _process_chunk_with_cost_injection(chunk: bytes, model_name: str) -> bytes: ...
+    def _process_chunk_with_cost_injection(
+        chunk: bytes, model_name: str, litellm_logging_obj: LiteLLMLoggingObj | None = None
+    ) -> bytes: ...
 
     @overload
     @staticmethod
-    def _process_chunk_with_cost_injection(chunk: object, model_name: str) -> object: ...
+    def _process_chunk_with_cost_injection(
+        chunk: object, model_name: str, litellm_logging_obj: LiteLLMLoggingObj | None = None
+    ) -> object: ...
 
     @staticmethod
-    def _process_chunk_with_cost_injection(chunk: object, model_name: str) -> object:
+    def _process_chunk_with_cost_injection(
+        chunk: object, model_name: str, litellm_logging_obj: LiteLLMLoggingObj | None = None
+    ) -> object:
         """
         Process a streaming chunk and inject cost information if enabled.
 
         Args:
             chunk: The streaming chunk (dict, str, bytes, or bytearray)
             model_name: Model name for cost calculation
+            litellm_logging_obj: The call's logging object, used for pricing
 
         Returns:
             The processed chunk with cost information injected if applicable
@@ -3441,21 +3499,27 @@ class ProxyBaseLLMRequestProcessing:
 
         try:
             if isinstance(chunk, dict):
-                maybe_modified: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(chunk, model_name)
+                maybe_modified: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(
+                    chunk, model_name, litellm_logging_obj
+                )
                 if maybe_modified is not None:
                     return maybe_modified
             elif isinstance(chunk, (bytes, bytearray)):
                 try:
                     s: Final = chunk.decode("utf-8")
                     if s.endswith(("\n\n", "\r\n\r\n")):
-                        maybe_mod = ProxyBaseLLMRequestProcessing._inject_cost_into_sse_frame_str(s, model_name)
+                        maybe_mod = ProxyBaseLLMRequestProcessing._inject_cost_into_sse_frame_str(
+                            s, model_name, litellm_logging_obj
+                        )
                         if maybe_mod is not None:
                             return maybe_mod.encode("utf-8")
                 except Exception:
                     pass
             elif isinstance(chunk, str):
                 # Try to parse SSE frame and inject cost into the data line
-                maybe_mod = ProxyBaseLLMRequestProcessing._inject_cost_into_sse_frame_str(chunk, model_name)
+                maybe_mod = ProxyBaseLLMRequestProcessing._inject_cost_into_sse_frame_str(
+                    chunk, model_name, litellm_logging_obj
+                )
                 if maybe_mod is not None:
                     # Ensure trailing frame separator
                     return maybe_mod if maybe_mod.endswith("\n\n") else (maybe_mod + "\n\n")
@@ -3466,13 +3530,16 @@ class ProxyBaseLLMRequestProcessing:
         return chunk
 
     @staticmethod
-    def _inject_cost_into_sse_frame_str(frame_str: str, model_name: str) -> str | None:
+    def _inject_cost_into_sse_frame_str(
+        frame_str: str, model_name: str, litellm_logging_obj: LiteLLMLoggingObj | None = None
+    ) -> str | None:
         """
         Inject cost information into an SSE frame string by modifying the JSON in the 'data:' line.
 
         Args:
             frame_str: SSE frame string that may contain multiple lines
             model_name: Model name for cost calculation
+            litellm_logging_obj: The call's logging object, forwarded for pricing
 
         Returns:
             Modified SSE frame string with cost injected, or None if no modification needed
@@ -3486,7 +3553,9 @@ class ProxyBaseLLMRequestProcessing:
                     json_part = stripped_ln.split("data:", 1)[1].strip()
                     if json_part and json_part != "[DONE]":
                         obj = json.loads(json_part)
-                        maybe_modified = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(obj, model_name)
+                        maybe_modified = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(
+                            obj, model_name, litellm_logging_obj
+                        )
                         if maybe_modified is not None:
                             lines[idx] = "data: " + safe_dumps(maybe_modified) + ("\r" if ln.endswith("\r") else "")
                             return "\n".join(lines)
@@ -3494,34 +3563,6 @@ class ProxyBaseLLMRequestProcessing:
         except Exception:
             return None
 
-    @staticmethod
-    def _anthropic_stream_usage_kwargs(usage: Mapping[str, Any]) -> Mapping[str, Any]:
-        prompt_tokens: Final = int(usage.get("input_tokens", 0) or 0)
-        completion_tokens: Final = int(usage.get("output_tokens", 0) or 0)
-        total_tokens: Final = int(
-            usage.get("total_tokens", prompt_tokens + completion_tokens) or (prompt_tokens + completion_tokens)
-        )
-        web_search_requests: Final = usage.get("web_search_requests")
-        server_tool_use: Final = (
-            ServerToolUse(web_search_requests=web_search_requests) if web_search_requests is not None else None
-        )
-        return MappingProxyType(
-            {
-                key: value
-                for key, value in (
-                    ("prompt_tokens", prompt_tokens),
-                    ("completion_tokens", completion_tokens),
-                    ("total_tokens", total_tokens),
-                    ("completion_tokens_details", usage.get("completion_tokens_details")),
-                    ("prompt_tokens_details", usage.get("prompt_tokens_details")),
-                    ("cache_creation_input_tokens", usage.get("cache_creation_input_tokens")),
-                    ("cache_read_input_tokens", usage.get("cache_read_input_tokens")),
-                    ("server_tool_use", server_tool_use),
-                )
-                if value is not None
-            }
-        )
-
     @staticmethod
     def _openai_stream_usage_kwargs(usage: Mapping[str, Any]) -> Mapping[str, Any]:
         prompt_tokens: Final = int(usage.get("prompt_tokens", 0) or 0)
@@ -3544,11 +3585,13 @@ class ProxyBaseLLMRequestProcessing:
         )
 
     @staticmethod
-    def _stream_usage_kwargs_for_event(obj: Mapping[str, object], usage: Mapping[str, Any]) -> Mapping[str, Any] | None:
+    def _stream_usage_for_event(obj: Mapping[str, object], usage: Mapping[str, Any]) -> Usage | None:
+        # Anthropic reports input_tokens excluding cache tokens, so reuse the non-streaming
+        # transformation to total the prompt and keep the 5m/1h cache creation split
         if obj.get("type") == "message_delta":
-            return ProxyBaseLLMRequestProcessing._anthropic_stream_usage_kwargs(usage)
+            return AnthropicConfig().calculate_usage(usage_object=usage, reasoning_content=None)
         if obj.get("object") == "chat.completion.chunk":
-            return ProxyBaseLLMRequestProcessing._openai_stream_usage_kwargs(usage)
+            return Usage(**ProxyBaseLLMRequestProcessing._openai_stream_usage_kwargs(usage))
         return None
 
     @staticmethod
@@ -3563,7 +3606,54 @@ class ProxyBaseLLMRequestProcessing:
             return None
 
     @staticmethod
-    def _inject_cost_into_usage_dict(obj: dict, model_name: str) -> dict | None:
+    def _logging_obj_cost_or_none(
+        model_response: ModelResponse, litellm_logging_obj: LiteLLMLoggingObj
+    ) -> float | None:
+        # Pricing a frame stamps cost_breakdown and, on failure, the cost-failure debug key onto
+        # the live logging object. The pass-through handlers never recompute either one, so a
+        # frame-derived breakdown would outlive the stream and land in the spend log. Snapshot
+        # both and put them back, so pricing here stays a read as far as the request is concerned
+        breakdown_before: Final = getattr(litellm_logging_obj, "cost_breakdown", None)
+        call_details: Final = getattr(litellm_logging_obj, "model_call_details", None)
+        debug_key: Final = "response_cost_failure_debug_information"
+        debug_missing: Final = object()
+        debug_before: Final = call_details.get(debug_key, debug_missing) if isinstance(call_details, dict) else None
+        try:
+            cost: Final = litellm_logging_obj._response_cost_calculator(result=model_response)  # pyright: ignore[reportPrivateUsage]  # reuse the call's own cost calc for pricing parity with the logging callback
+        except Exception:  # noqa: BLE001  # a pricing failure falls back to model-name pricing instead of breaking the stream
+            return None
+        finally:
+            if hasattr(litellm_logging_obj, "cost_breakdown"):
+                litellm_logging_obj.cost_breakdown = breakdown_before
+            if isinstance(call_details, dict):
+                if debug_before is debug_missing:
+                    call_details.pop(debug_key, None)
+                else:
+                    call_details[debug_key] = debug_before
+        return float(cost) if isinstance(cost, (int, float)) and not isinstance(cost, bool) else None
+
+    @staticmethod
+    def _streamed_usage_cost(
+        model_response: ModelResponse,
+        model_name: str,
+        service_tier: str | None,
+        litellm_logging_obj: LiteLLMLoggingObj | None,
+    ) -> float | None:
+        # Pricing via the logging object inherits the deployment's custom pricing, so the
+        # streamed cost matches what the logging callback records instead of sticker price
+        cost_from_logging_obj: Final = (
+            ProxyBaseLLMRequestProcessing._logging_obj_cost_or_none(model_response, litellm_logging_obj)
+            if litellm_logging_obj is not None
+            else None
+        )
+        if cost_from_logging_obj is not None:
+            return cost_from_logging_obj
+        return ProxyBaseLLMRequestProcessing._completion_cost_or_none(model_response, model_name, service_tier)
+
+    @staticmethod
+    def _inject_cost_into_usage_dict(
+        obj: dict, model_name: str, litellm_logging_obj: LiteLLMLoggingObj | None = None
+    ) -> dict | None:
         """
         Inject cost information into the usage object of a streamed usage event
         (Anthropic ``message_delta`` or OpenAI ``chat.completion.chunk``).
@@ -3571,6 +3661,7 @@ class ProxyBaseLLMRequestProcessing:
         Args:
             obj: Dictionary containing the SSE event data
             model_name: Model name for cost calculation
+            litellm_logging_obj: The call's logging object, used for pricing
 
         Returns:
             Modified dictionary with cost injected, or None if no modification needed
@@ -3578,14 +3669,15 @@ class ProxyBaseLLMRequestProcessing:
         usage: Final = obj.get("usage")
         if not isinstance(usage, dict):
             return None
-        usage_kwargs: Final = ProxyBaseLLMRequestProcessing._stream_usage_kwargs_for_event(obj, usage)
-        if usage_kwargs is None:
+        stream_usage: Final = ProxyBaseLLMRequestProcessing._stream_usage_for_event(obj, usage)
+        if stream_usage is None:
             return None
         service_tier: Final = obj.get("service_tier")
-        cost_val: Final = ProxyBaseLLMRequestProcessing._completion_cost_or_none(
-            ModelResponse(usage=Usage(**usage_kwargs)),
+        cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
+            ModelResponse(usage=stream_usage),
             model_name,
             service_tier if isinstance(service_tier, str) else None,
+            litellm_logging_obj,
         )
         if cost_val is None:
             return None

litellm/proxy/common_utils/reset_budget_job.py

@@ -4,6 +4,7 @@ import time
 from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
 from dataclasses import dataclass
 from datetime import datetime, timezone
+from enum import Enum
 from types import MappingProxyType
 from typing import Final, Literal, Protocol, TypeVar, assert_never
 
@@ -14,7 +15,9 @@ from litellm.constants import (
     GLOBAL_PROXY_SPEND_CACHE_KEY,
     LITELLM_PROXY_BUDGET_NAME,
     RESET_BUDGET_JOB_BATCH_SIZE,
+    RESET_BUDGET_JOB_LOCK_TTL_SECONDS,
     RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN,
+    RESET_BUDGET_JOB_NAME,
 )
 from litellm.proxy._types import (
     DB_RETRY_SAFE_ERROR_TYPES,
@@ -30,6 +33,7 @@ from litellm.proxy.common_utils.timezone_utils import (
     get_budget_reset_settings,
 )
 from litellm.proxy.common_utils.user_api_key_cache import tag_cache_key
+from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
 from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 from litellm.repositories.organization_repository import OrganizationRepository
@@ -195,12 +199,94 @@ async def _run_phase_in_chunks(process_chunk: Callable[[], Awaitable[_ChunkOutco
             return
 
 
+@dataclass(frozen=True, slots=True)
+class _LazyJson:
+    """Serialize only if a log record is actually emitted.
+
+    ``logger.debug("... %s", json.dumps(rows))`` evaluates the dump before the
+    logger decides to drop the record, so a chunk of rows is serialized on the
+    event loop on every tick at any log level. Passing this instead defers the
+    work to the formatter.
+    """
+
+    value: object
+
+    def __str__(self) -> str:
+        return json.dumps(self.value, indent=4, default=str)
+
+
+class _Lease(Enum):
+    """Whether this pod may sweep, and whether it owes a lock release."""
+
+    LEADER = "leader"
+    UNGUARDED = "unguarded"
+    FOLLOWER = "follower"
+
+
+async def _write_key_windows(prisma_client: PrismaClient, row_id: str, payload: str) -> None:
+    await VerificationTokenRepository(prisma_client).table.update(
+        where={"token": row_id},
+        data={"budget_limits": payload},
+    )
+
+
+async def _write_team_windows(prisma_client: PrismaClient, row_id: str, payload: str) -> None:
+    await TeamRepository(prisma_client).table.update(
+        where={"team_id": row_id},
+        data={"budget_limits": payload},
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class _WindowSource:
+    """A table whose rows carry their own per-window budget limits."""
+
+    table: str
+    id_column: str
+    counter_prefix: str
+    log_subject: str
+    retry_subject: str
+    write: Callable[[PrismaClient, str, str], Awaitable[None]]
+
+    def page_query(self) -> str:
+        """One keyset page, ordered by the primary key so the cursor never repeats a row.
+
+        prisma-client-python cannot null-filter a ``Json?`` column (no DbNull /
+        JsonNull sentinel, RobertCraigie/prisma-client-py#714), so the read stays
+        raw SQL; the table and column names are module constants, never input.
+        Writes still go through the ORM.
+        """
+        return (
+            f'SELECT {self.id_column}, budget_limits FROM "{self.table}" '
+            f"WHERE budget_limits IS NOT NULL AND {self.id_column} > $1 "
+            f"ORDER BY {self.id_column} LIMIT $2"
+        )
+
+
+_WINDOW_SOURCES: Final[tuple[_WindowSource, ...]] = (
+    _WindowSource(
+        table="LiteLLM_VerificationToken",
+        id_column="token",
+        counter_prefix="spend:key",
+        log_subject="keys",
+        retry_subject="key",
+        write=_write_key_windows,
+    ),
+    _WindowSource(
+        table="LiteLLM_TeamTable",
+        id_column="team_id",
+        counter_prefix="spend:team",
+        log_subject="teams",
+        retry_subject="team",
+        write=_write_team_windows,
+    ),
+)
+
+
 def _budget_cascade_event_metadata(cascade: _BudgetCascade) -> dict[str, object]:
     return {
         "num_budgets_found": len(cascade.budgets),
-        "budgets_found": json.dumps(cascade.budgets, indent=4, default=str),
         "num_endusers_found": len(cascade.endusers),
-        "endusers_found": json.dumps(cascade.endusers, indent=4, default=str),
     }
 
 
@@ -214,10 +300,61 @@ class ResetBudgetJob:
         proxy_logging_obj: ProxyLogging,
         prisma_client: PrismaClient,
         reset_settings: BudgetResetSettings | None = None,
+        pod_lock_manager: PodLockManager | None = None,
     ):
         self.proxy_logging_obj: ProxyLogging = proxy_logging_obj
         self.prisma_client: PrismaClient = prisma_client
         self.reset_settings: BudgetResetSettings = reset_settings or get_budget_reset_settings()
+        self.pod_lock_manager: PodLockManager | None = pod_lock_manager
+
+    async def _lease_is_held(self, lock_manager: PodLockManager) -> bool:
+        """True only when the lease is readable and someone holds it.
+
+        An unreadable lock reports as unheld so the caller sweeps rather than
+        skipping; being wrong here costs a duplicate sweep, and the alternative
+        strands every expired budget at its cap.
+        """
+        if lock_manager.redis_cache is None:
+            return False
+        try:
+            lock_key: Final = lock_manager.get_redis_lock_key(RESET_BUDGET_JOB_NAME)
+            return bool(await lock_manager.redis_cache.async_get_cache(lock_key))
+        except Exception as exc:  # noqa: BLE001  # an unreadable lease must not strand the sweep
+            verbose_proxy_logger.warning("Reset budget job: could not read the reset lease: %s", exc)
+            return False
+
+    async def _acquire_lease(self) -> _Lease:
+        """Elect one sweeper per tick.
+
+        Every pod schedules this job, and each one otherwise re-reads the whole
+        due population and writes it back at the same calendar boundary, so a
+        fleet multiplies one sweep's Postgres load by its replica count. A
+        deployment with no Redis-backed lock manager runs unguarded, as it
+        always has.
+        """
+        lock_manager: Final = self.pod_lock_manager
+        if lock_manager is None or lock_manager.redis_cache is None:
+            return _Lease.UNGUARDED
+
+        if await lock_manager.acquire_lock(
+            cronjob_id=RESET_BUDGET_JOB_NAME,
+            ttl=RESET_BUDGET_JOB_LOCK_TTL_SECONDS,
+        ):
+            return _Lease.LEADER
+
+        if await self._lease_is_held(lock_manager):
+            verbose_proxy_logger.debug("Reset budget job: another pod holds the reset lease, skipping this tick")
+            return _Lease.FOLLOWER
+
+        # acquire_lock reports contention and an unreachable Redis identically, so
+        # treating a failed acquire as contention would skip the sweep on every pod
+        # at once for as long as Redis is down. Sweeping unguarded costs duplicate
+        # work; not sweeping leaves every expired budget pinned at its cap.
+        verbose_proxy_logger.warning(
+            "Reset budget job: could not take the reset lease and no other pod holds it, "
+            "sweeping unguarded rather than skipping the tick"
+        )
+        return _Lease.UNGUARDED
 
     async def reset_budget(
         self,
@@ -228,15 +365,25 @@ class ResetBudgetJob:
         Resets their spend
 
         Updates db
+
+        Runs on one pod per tick where a Redis lease is available.
         """
         if self.prisma_client is None:
             return
 
-        await self.reset_budget_for_litellm_keys()
-        await self.reset_budget_for_litellm_users()
-        await self.reset_budget_for_litellm_teams()
-        await self.reset_budget_for_litellm_budget_table()
-        await self.reset_budget_windows()
+        lease: Final = await self._acquire_lease()
+        if lease is _Lease.FOLLOWER:
+            return
+
+        try:
+            await self.reset_budget_for_litellm_keys()
+            await self.reset_budget_for_litellm_users()
+            await self.reset_budget_for_litellm_teams()
+            await self.reset_budget_for_litellm_budget_table()
+            await self.reset_budget_windows()
+        finally:
+            if lease is _Lease.LEADER and self.pod_lock_manager is not None:
+                await self.pod_lock_manager.release_lock(cronjob_id=RESET_BUDGET_JOB_NAME)
 
     async def _with_db_retry(self, operation: Callable[[], Awaitable[_RowT]], *, reason: str) -> _RowT:
         """Reconnect and retry once on a transport error, so a dropped connection
@@ -647,7 +794,7 @@ class ResetBudgetJob:
                 ),
                 reason="reset_budget_read_keys_failure",
             )
-            verbose_proxy_logger.debug("Keys to reset %s", json.dumps(keys_to_reset, indent=4, default=str))
+            verbose_proxy_logger.debug("Keys to reset %s", _LazyJson(keys_to_reset))
             updated_keys: Final[list[LiteLLM_VerificationToken]] = []
             failed_keys: Final = []
             if keys_to_reset is not None and len(keys_to_reset) > 0:
@@ -666,7 +813,7 @@ class ResetBudgetJob:
                         failed_keys.append({"key": key, "error": str(e)})
                         verbose_proxy_logger.exception("Failed to reset budget for key: %s", key)
 
-                verbose_proxy_logger.debug("Updated keys %s", json.dumps(updated_keys, indent=4, default=str))
+                verbose_proxy_logger.debug("Updated keys %s", _LazyJson(updated_keys))
 
                 if updated_keys:
                     await self._write_key_reset_updates(updated_keys=updated_keys)
@@ -691,7 +838,6 @@ class ResetBudgetJob:
                     end_time=end_time,
                     event_metadata={
                         "num_keys_found": len(keys_to_reset) if keys_to_reset else 0,
-                        "keys_found": json.dumps(keys_to_reset, indent=4, default=str),
                     },
                 )
                 return outcome
@@ -705,11 +851,8 @@ class ResetBudgetJob:
                     end_time=end_time,
                     event_metadata={
                         "num_keys_found": len(keys_to_reset) if keys_to_reset else 0,
-                        "keys_found": json.dumps(keys_to_reset, indent=4, default=str),
                         "num_keys_updated": len(updated_keys),
-                        "keys_updated": json.dumps(updated_keys, indent=4, default=str),
                         "num_keys_failed": len(failed_keys),
-                        "keys_failed": json.dumps(failed_keys, indent=4, default=str),
                     },
                 )
             )
@@ -725,7 +868,6 @@ class ResetBudgetJob:
                     end_time=end_time,
                     event_metadata={
                         "num_keys_found": len(keys_to_reset) if keys_to_reset else 0,
-                        "keys_found": json.dumps(keys_to_reset, indent=4, default=str),
                     },
                 )
             )
@@ -777,7 +919,7 @@ class ResetBudgetJob:
                         failed_users.append({"user": user, "error": str(e)})
                         verbose_proxy_logger.exception("Failed to reset budget for user: %s", user)
 
-                verbose_proxy_logger.debug("Updated users %s", json.dumps(updated_users, indent=4, default=str))
+                verbose_proxy_logger.debug("Updated users %s", _LazyJson(updated_users))
                 if updated_users:
                     await self._write_user_reset_updates(updated_users=updated_users)
                     for u in updated_users:
@@ -805,7 +947,6 @@ class ResetBudgetJob:
                     end_time=end_time,
                     event_metadata={
                         "num_users_found": len(users_to_reset) if users_to_reset else 0,
-                        "users_found": json.dumps(users_to_reset, indent=4, default=str),
                     },
                 )
                 return outcome
@@ -819,11 +960,8 @@ class ResetBudgetJob:
                     end_time=end_time,
                     event_metadata={
                         "num_users_found": len(users_to_reset) if users_to_reset else 0,
-                        "users_found": json.dumps(users_to_reset, indent=4, default=str),
                         "num_users_updated": len(updated_users),
-                        "users_updated": json.dumps(updated_users, indent=4, default=str),
                         "num_users_failed": len(failed_users),
-                        "users_failed": json.dumps(failed_users, indent=4, default=str),
                     },
                 )
             )
@@ -839,7 +977,6 @@ class ResetBudgetJob:
                     end_time=end_time,
                     event_metadata={
                         "num_users_found": len(users_to_reset) if users_to_reset else 0,
-                        "users_found": json.dumps(users_to_reset, indent=4, default=str),
                     },
                 )
             )
@@ -891,7 +1028,7 @@ class ResetBudgetJob:
                         failed_teams.append({"team": team, "error": str(e)})
                         verbose_proxy_logger.exception("Failed to reset budget for team: %s", team)
 
-                verbose_proxy_logger.debug("Updated teams %s", json.dumps(updated_teams, indent=4, default=str))
+                verbose_proxy_logger.debug("Updated teams %s", _LazyJson(updated_teams))
                 if updated_teams:
                     await self._write_team_reset_updates(updated_teams=updated_teams)
                     for t in updated_teams:
@@ -917,7 +1054,6 @@ class ResetBudgetJob:
                     end_time=end_time,
                     event_metadata={
                         "num_teams_found": len(teams_to_reset) if teams_to_reset else 0,
-                        "teams_found": json.dumps(teams_to_reset, indent=4, default=str),
                     },
                 )
                 return outcome
@@ -931,11 +1067,8 @@ class ResetBudgetJob:
                     end_time=end_time,
                     event_metadata={
                         "num_teams_found": len(teams_to_reset) if teams_to_reset else 0,
-                        "teams_found": json.dumps(teams_to_reset, indent=4, default=str),
                         "num_teams_updated": len(updated_teams),
-                        "teams_updated": json.dumps(updated_teams, indent=4, default=str),
                         "num_teams_failed": len(failed_teams),
-                        "teams_failed": json.dumps(failed_teams, indent=4, default=str),
                     },
                 )
             )
@@ -951,7 +1084,6 @@ class ResetBudgetJob:
                     end_time=end_time,
                     event_metadata={
                         "num_teams_found": len(teams_to_reset) if teams_to_reset else 0,
-                        "teams_found": json.dumps(teams_to_reset, indent=4, default=str),
                     },
                 )
             )
@@ -995,82 +1127,82 @@ class ResetBudgetJob:
         from litellm.proxy.proxy_server import spend_counter_cache
 
         now: Final = datetime.utcnow()
+        for source in _WINDOW_SOURCES:
+            try:
+                await self._reset_windows_for(source=source, now=now, spend_counter_cache=spend_counter_cache)
+            except Exception as e:
+                verbose_proxy_logger.exception("Failed to reset budget windows for %s: %s", source.log_subject, e)
 
-        # Note on raw SQL: prisma-client-python does not support null-filtering
-        # on `Json?` columns (no DbNull/JsonNull sentinel — see
-        # RobertCraigie/prisma-client-py#714). We use `query_raw` with
-        # `IS NOT NULL` so we don't materialize every key/team row on each
-        # tick of the reset job. Writes still go through the ORM.
+    async def _reset_windows_for(
+        self,
+        source: _WindowSource,
+        now: datetime,
+        spend_counter_cache: DualCache,
+    ) -> None:
+        """Walk one table's windowed rows a page at a time, to the end.
 
-        # --- Keys ---
-        try:
-            key_rows: Final = await self._with_db_retry(
-                lambda: self.prisma_client.db.query_raw(
-                    'SELECT token, budget_limits FROM "LiteLLM_VerificationToken" WHERE budget_limits IS NOT NULL'
-                ),
-                reason="reset_budget_read_key_windows_failure",
+        Paging is what bounds the memory: the previous form pulled every row
+        carrying budget_limits into one result set on every tick, which grows
+        with the deployment's key count and is paid on the event loop.
+
+        The walk deliberately has no per-run page cap. A cap has to remember
+        where it stopped, and that position cannot live in the process: the
+        lease is released after each sweep, so the next tick can elect a
+        different pod whose own position is unset. It would restart at the first
+        row and never reach the tail, pinning those windows at their cap for
+        good. The cursor strictly advances, so the walk terminates on its own
+        without needing a bound.
+        """
+        cursor = ""
+        while True:
+            next_cursor = await self._reset_window_page(
+                source=source,
+                cursor=cursor,
+                now=now,
+                spend_counter_cache=spend_counter_cache,
             )
-            for row in key_rows:
-                raw = row["budget_limits"]
-                if not raw:
-                    continue
-                windows: list = raw if isinstance(raw, list) else json.loads(raw)
-                changed = False
-                for window in windows:
-                    counter_key = f"spend:key:{row['token']}:window:{window['budget_duration']}"
-                    if await ResetBudgetJob._reset_expired_window(
-                        window,
-                        counter_key,
-                        spend_counter_cache,
-                        now,
-                        self.reset_settings,
-                    ):
-                        changed = True
-                if changed:
-                    await self._with_db_write_retry(
-                        lambda: VerificationTokenRepository(self.prisma_client).table.update(
-                            where={"token": row["token"]},
-                            data={"budget_limits": json.dumps(windows)},
-                        ),
-                        reason="reset_budget_write_key_windows_failure",
-                    )
-        except Exception as e:
-            verbose_proxy_logger.exception("Failed to reset budget windows for keys: %s", e)
+            if next_cursor is None:
+                return
+            cursor = next_cursor
 
-        # --- Teams ---
-        try:
-            team_rows: Final = await self._with_db_retry(
-                lambda: self.prisma_client.db.query_raw(
-                    'SELECT team_id, budget_limits FROM "LiteLLM_TeamTable" WHERE budget_limits IS NOT NULL'
-                ),
-                reason="reset_budget_read_team_windows_failure",
-            )
-            for row in team_rows:
-                raw = row["budget_limits"]
-                if not raw:
-                    continue
-                windows = raw if isinstance(raw, list) else json.loads(raw)
-                changed = False
-                for window in windows:
-                    counter_key = f"spend:team:{row['team_id']}:window:{window['budget_duration']}"
-                    if await ResetBudgetJob._reset_expired_window(
-                        window,
-                        counter_key,
-                        spend_counter_cache,
-                        now,
-                        self.reset_settings,
-                    ):
-                        changed = True
-                if changed:
-                    await self._with_db_write_retry(
-                        lambda: TeamRepository(self.prisma_client).table.update(
-                            where={"team_id": row["team_id"]},
-                            data={"budget_limits": json.dumps(windows)},
-                        ),
-                        reason="reset_budget_write_team_windows_failure",
-                    )
-        except Exception as e:
-            verbose_proxy_logger.exception("Failed to reset budget windows for teams: %s", e)
+    async def _reset_window_page(
+        self,
+        source: _WindowSource,
+        cursor: str,
+        now: datetime,
+        spend_counter_cache: DualCache,
+    ) -> str | None:
+        """Reset one page of windows; return the next cursor, or None when drained."""
+        rows: Final = await self._with_db_retry(
+            lambda: self.prisma_client.db.query_raw(source.page_query(), cursor, RESET_BUDGET_JOB_BATCH_SIZE),
+            reason=f"reset_budget_read_{source.retry_subject}_windows_failure",
+        )
+        for row in rows:
+            raw = row["budget_limits"]
+            if not raw:
+                continue
+            row_id: str = row[source.id_column]
+            windows: list = raw if isinstance(raw, list) else json.loads(raw)
+            changed = False
+            for window in windows:
+                counter_key = f"{source.counter_prefix}:{row_id}:window:{window['budget_duration']}"
+                if await ResetBudgetJob._reset_expired_window(
+                    window,
+                    counter_key,
+                    spend_counter_cache,
+                    now,
+                    self.reset_settings,
+                ):
+                    changed = True
+            if changed:
+                await self._with_db_write_retry(
+                    lambda: source.write(self.prisma_client, row_id, json.dumps(windows)),
+                    reason=f"reset_budget_write_{source.retry_subject}_windows_failure",
+                )
+
+        if len(rows) < RESET_BUDGET_JOB_BATCH_SIZE:
+            return None
+        return rows[-1][source.id_column]
 
     @staticmethod
     async def _reset_budget_common(

litellm/proxy/management_endpoints/cost_tracking_settings.py

@@ -15,6 +15,7 @@ from dataclasses import dataclass
 from typing import Final
 
 from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -439,6 +440,76 @@ async def update_cost_margin_config(
         )
 
 
+class BlockUnpricedModelsRequest(BaseModel):
+    enabled: bool
+
+
+class BlockUnpricedModelsResponse(BaseModel):
+    enabled: bool
+
+
+@router.get(
+    "/config/block_requests_for_models_without_pricing",
+    tags=("Cost Tracking",),
+    dependencies=(Depends(user_api_key_auth),),
+    response_model=BlockUnpricedModelsResponse,
+)
+async def get_block_requests_for_models_without_pricing() -> BlockUnpricedModelsResponse:
+    return BlockUnpricedModelsResponse(enabled=bool(litellm.block_requests_for_models_without_pricing))
+
+
+@router.patch(
+    "/config/block_requests_for_models_without_pricing",
+    tags=("Cost Tracking",),
+    dependencies=(Depends(user_api_key_auth),),
+    response_model=BlockUnpricedModelsResponse,
+)
+async def update_block_requests_for_models_without_pricing(
+    request: BlockUnpricedModelsRequest,
+) -> BlockUnpricedModelsResponse:
+    from litellm.proxy.proxy_server import (
+        prisma_client,
+        proxy_config,
+        store_model_in_db,
+    )
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500,
+            detail={  # mutable-ok: HTTPException detail must be a plain mapping
+                "error": CommonProxyErrors.db_not_connected_error.value
+            },
+        )
+
+    if store_model_in_db is not True:
+        raise HTTPException(
+            status_code=500,
+            detail={  # mutable-ok: HTTPException detail must be a plain mapping
+                "error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."
+            },
+        )
+
+    try:
+        config = await proxy_config.get_config()
+        if "litellm_settings" not in config:
+            config["litellm_settings"] = {}  # mutable-ok: config is a plain-dict payload for save_config
+        config["litellm_settings"]["block_requests_for_models_without_pricing"] = request.enabled
+        await proxy_config.save_config(new_config=config)
+
+        litellm.block_requests_for_models_without_pricing = request.enabled
+        verbose_proxy_logger.info("Updated block_requests_for_models_without_pricing: %s", request.enabled)
+
+        return BlockUnpricedModelsResponse(enabled=request.enabled)
+    except Exception as e:  # noqa: BLE001  # any config persistence failure must surface as a 500 response, not a crash
+        verbose_proxy_logger.error("Error updating block_requests_for_models_without_pricing: %s", e)
+        raise HTTPException(
+            status_code=500,
+            detail={  # mutable-ok: HTTPException detail must be a plain mapping
+                "error": f"Failed to update setting: {e!s}"
+            },
+        )
+
+
 @router.post(
     "/cost/estimate",
     tags=["Cost Tracking"],

litellm/proxy/management_endpoints/scim/scim_v2.py

@@ -487,13 +487,18 @@ class _UnknownMember(NamedTuple):
     value: str
 
 
-_ClassifiedGroupMember = Union[_ResolvedUserMember, _SkippedGroupMember, _UnknownMember]
+class _AmbiguousMember(NamedTuple):
+    value: str
+
+
+_ClassifiedGroupMember = Union[_ResolvedUserMember, _SkippedGroupMember, _UnknownMember, _AmbiguousMember]
 
 
 class _PartitionedMembers(NamedTuple):
     resolved_ids: tuple[str, ...]
     skipped: tuple[_SkippedGroupMember, ...]
     unknown_ids: tuple[str, ...]
+    ambiguous_values: tuple[str, ...]
 
 
 def _member_value(member: SCIMMember) -> str:
@@ -536,6 +541,44 @@ def _team_metadata_has_scim_provenance(team_metadata: object) -> bool:
     return bool(fields.get(SCIM_MANAGED_TEAM_METADATA_KEY)) or fields.get(SCIM_TEAM_DATA_METADATA_KEY) is not None
 
 
+class _CaseInsensitiveMatch(TypedDict):
+    equals: ReadOnly[str]
+    mode: ReadOnly[str]
+
+
+async def _users_named_by_member_value(
+    value: str, prisma_client: PrismaClient, *, take: int | None = 2
+) -> tuple[str, ...]:
+    """Every user id this member value names, by SSO identity or by email.
+
+    Both fields are searched in one pass, because searching either first would hide a
+    value that names one account by its SSO identity and another by its email, and
+    hand the group to whichever field was searched first.
+
+    They are not compared alike. An email is matched the way ``new_user`` matches one
+    before it accepts a new account, case-insensitively: matching more strictly than
+    the layer that would reject the placeholder is what turned a member id whose
+    casing differed from the stored email into a 500 on the whole push. An SSO
+    identity is matched exactly, because OIDC defines ``sub`` as case-sensitive and
+    nothing folds its case on the way in, so treating two subjects that differ in case
+    as one would hand the group to an account the provider never named.
+
+    ``take`` bounds the read for a caller that only needs to know whether the value
+    names one account or several; ``user_email`` carries no index, so letting the scan
+    stop early is worth the two rows. A caller that has to know *which* accounts, as a
+    removal does, passes None. That set is the accounts sharing one identity, which is
+    a handful at worst.
+    """
+    subject: Final = value.strip()
+    email: Final[_CaseInsensitiveMatch] = {"equals": subject, "mode": "insensitive"}
+    rows: Final = await _table(UserRepository(prisma_client)).find_many(
+        # mutable-ok: the Prisma serializer requires concrete dicts and a concrete list
+        where={"OR": [{"sso_user_id": subject}, {"user_email": email}]},
+        take=take,
+    )
+    return tuple(dict.fromkeys(row.user_id for row in rows))
+
+
 async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient) -> _ClassifiedGroupMember:
     """
     Decide what a single SCIM group member refers to.
@@ -557,6 +600,20 @@ async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient
       one the identity provider writes. An id the IdP called a User is a user
       even if some team happens to share the id, and a team created here rather
       than through SCIM is not evidence of anything about the member.
+
+    When those checks miss on an otherwise user-shaped member, its value is looked
+    up as an SSO identity or an email, and a match resolves to that user's
+    ``user_id``. A value that names more than one account is ambiguous rather than
+    unknown: it names a real person we cannot identify, so it is neither guessed at
+    nor provisioned.
+
+    An exact ``user_id`` hit is checked the same way rather than trusted outright. A
+    value can be one account's id and another's SSO identity or email, and taking the
+    id on sight would hand the group to whichever account happened to be keyed by it.
+    The placeholders this bug provisioned are that shape exactly, since they are keyed
+    by the very id the provider keeps pushing, so on a tenant that already has them
+    the membership is refused and named rather than silently landing on the
+    placeholder again.
     """
     value: Final = _member_value(member)
     member_type: Final = _normalized_member_type(member)
@@ -566,6 +623,18 @@ async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient
 
     user: Final = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": value})
     if user is not None:
+        shared_with: Final = tuple(
+            other for other in await _users_named_by_member_value(value, prisma_client) if other != value
+        )
+        if shared_with:
+            verbose_proxy_logger.warning(
+                "SCIM: group member '%s' is one account's user id and is also account '%s' by SSO identity or email, "
+                "so the membership cannot be attributed. A placeholder an earlier release provisioned under this id "
+                "looks exactly like this and should be deleted so the real account can be matched",
+                value,
+                shared_with[0],
+            )
+            return _AmbiguousMember(value=value)
         return _ResolvedUserMember(user_id=value)
 
     if member_type is not None and member_type != "user":
@@ -576,6 +645,22 @@ async def _classify_group_member(member: SCIMMember, prisma_client: PrismaClient
         if team is not None and _team_metadata_has_scim_provenance(team.metadata):
             return _SkippedGroupMember(value=value, reason="existing_team")
 
+    named: Final = await _users_named_by_member_value(value, prisma_client)
+    if len(named) == 1:
+        verbose_proxy_logger.info(
+            "SCIM: group member '%s' matched user_id '%s' by SSO identity or email",
+            value,
+            named[0],
+        )
+        return _ResolvedUserMember(user_id=named[0])
+    if len(named) > 1:
+        verbose_proxy_logger.warning(
+            "SCIM: group member '%s' names more than one account by SSO identity or email and cannot be resolved "
+            "unambiguously",
+            value,
+        )
+        return _AmbiguousMember(value=value)
+
     return _UnknownMember(value=value)
 
 
@@ -583,11 +668,13 @@ def _bucketed_member(entry: _ClassifiedGroupMember) -> _PartitionedMembers:
     """The single-member partition one classified entry contributes."""
     match entry:
         case _ResolvedUserMember(user_id=user_id):
-            return _PartitionedMembers(resolved_ids=(user_id,), skipped=(), unknown_ids=())
+            return _PartitionedMembers(resolved_ids=(user_id,), skipped=(), unknown_ids=(), ambiguous_values=())
         case _SkippedGroupMember():
-            return _PartitionedMembers(resolved_ids=(), skipped=(entry,), unknown_ids=())
+            return _PartitionedMembers(resolved_ids=(), skipped=(entry,), unknown_ids=(), ambiguous_values=())
         case _UnknownMember(value=value):
-            return _PartitionedMembers(resolved_ids=(), skipped=(), unknown_ids=(value,))
+            return _PartitionedMembers(resolved_ids=(), skipped=(), unknown_ids=(value,), ambiguous_values=())
+        case _AmbiguousMember(value=value):
+            return _PartitionedMembers(resolved_ids=(), skipped=(), unknown_ids=(), ambiguous_values=(value,))
         case _:
             assert_never(entry)
 
@@ -599,6 +686,7 @@ def _partition_classified_members(classified: Iterable[_ClassifiedGroupMember])
         resolved_ids=tuple(chain.from_iterable(bucket.resolved_ids for bucket in bucketed)),
         skipped=tuple(chain.from_iterable(bucket.skipped for bucket in bucketed)),
         unknown_ids=tuple(chain.from_iterable(bucket.unknown_ids for bucket in bucketed)),
+        ambiguous_values=tuple(chain.from_iterable(bucket.ambiguous_values for bucket in bucketed)),
     )
 
 
@@ -608,7 +696,7 @@ def _admitted_member_id(entry: _ClassifiedGroupMember, created_ids: frozenset[st
             return user_id
         case _UnknownMember(value=value):
             return value if value in created_ids else None
-        case _SkippedGroupMember():
+        case _SkippedGroupMember() | _AmbiguousMember():
             return None
         case _:
             assert_never(entry)
@@ -662,6 +750,70 @@ async def _ensure_group_member_user(
     raise HTTPException(status_code=500, detail=detail)
 
 
+def _roster_entries_named_by(value: str, roster: frozenset[str], resolved: tuple[str, ...]) -> tuple[str, ...]:
+    """The members of this group a removal value names.
+
+    Both ways of naming one count together. The id as written counts when the roster
+    holds it verbatim, which is how an earlier release recorded a member it could not
+    match, and the accounts it resolves to count when they are on the roster. Counting
+    only the resolved ones would let a value that is one member's canonical id and
+    another member's email revoke both, since each looks singular on its own.
+    """
+    return tuple(
+        dict.fromkeys(
+            chain(
+                (value,) if value in roster else (),
+                (user_id for user_id in resolved if user_id in roster),
+            )
+        )
+    )
+
+
+async def _member_ids_to_drop(
+    members: Sequence[SCIMMember], roster: frozenset[str], prisma_client: PrismaClient
+) -> frozenset[str]:
+    """The members a ``remove`` clears, one per id the request names.
+
+    The roster holds canonical user ids, so a directory that added someone by their
+    email or SSO identity has to be able to remove them by that same value, and a
+    member an earlier release recorded under the raw id has to stay removable by it.
+
+    Ambiguity is a property of the table as it stands, not of the value, so a value
+    that named one person when they were admitted can name two later. Resolving a
+    removal against the whole table would then drop nobody while answering 200, and
+    the person the directory just took out of the group would keep the team. So a
+    removal keeps only the accounts already on the roster: one is unambiguous however
+    many strangers share the address, none means there is nothing to revoke, and only
+    a value naming two of this group's own members is genuinely undecidable. That last
+    case fails rather than reporting a removal it did not perform, or revoking both.
+
+    Raises:
+        HTTPException: 400 when a member id names more than one current member.
+    """
+    written: Final = frozenset(_member_value(member) for member in members)
+    matched: Final = tuple(
+        [
+            (
+                value,
+                _roster_entries_named_by(
+                    value, roster, await _users_named_by_member_value(value, prisma_client, take=None)
+                ),
+            )
+            for value in sorted(written)
+        ]
+    )
+    undecidable: Final = tuple(value for value, entries in matched if len(entries) > 1)
+    if undecidable:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": f"Member ID '{undecidable[0]}' names more than one member of this group, so the removal "
+                "cannot be attributed. Send the LiteLLM user ID as the member value, or resolve the duplicate."
+            },
+        )
+    return frozenset(chain.from_iterable(entries for _, entries in matched))
+
+
 async def _resolve_group_member_ids(
     members: Sequence[SCIMMember],
     created_via: str,
@@ -670,17 +822,18 @@ async def _resolve_group_member_ids(
     """
     Resolve SCIM group members to LiteLLM user ids, dropping members that are not users.
 
-    Only the operations that put ids onto a roster resolve their members: an id
-    that resolves to nothing is created when litellm_settings.scim_upsert_user is
-    True (default) and rejected per SCIM 2.0 otherwise. Removals do not come
-    through here; dropping an id is idempotent, so it needs neither a lookup nor a
-    user to drop.
+    Member ids are matched by ``user_id`` first, then by SSO identity or email. An
+    id that resolves to nothing is created when litellm_settings.scim_upsert_user is
+    True (default) and rejected per SCIM 2.0 otherwise. Removals do not come through
+    here: they resolve through ``_member_ids_to_drop`` instead, which neither creates
+    a user nor fails on an id it cannot place.
 
     Raises:
-        HTTPException: 400 when a member id is empty, or when scim_upsert_user is
-        False and a member id is neither an existing user, an existing team, nor a
-        member declared to be something other than a user. 500 when a member's
-        user row can neither be created nor found.
+        HTTPException: 400 when a member id is empty, when a member id names more
+        than one user, or when scim_upsert_user is False and a member id is neither
+        an existing user, an existing team, nor a member declared to be something
+        other than a user. 500 when a member's user row can neither be created nor
+        found.
     """
     classified: Final = tuple([await _classify_group_member(member, prisma_client) for member in members])
     partition: Final = _partition_classified_members(classified)
@@ -692,6 +845,16 @@ async def _resolve_group_member_ids(
             skipped.reason,
         )
 
+    if partition.ambiguous_values:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": f"Member ID '{partition.ambiguous_values[0]}' names more than one LiteLLM user, so the "
+                "group membership cannot be attributed. Resolve the duplicate, which for an id that also matches a "
+                "SCIM-provisioned placeholder means deleting that placeholder."
+            },
+        )
+
     if partition.unknown_ids and not await _get_scim_upsert_user_setting():
         raise HTTPException(
             status_code=400,
@@ -702,6 +865,13 @@ async def _resolve_group_member_ids(
         )
 
     unique_unknown_ids: Final = tuple(dict.fromkeys(partition.unknown_ids))
+    for user_id in unique_unknown_ids:
+        verbose_proxy_logger.warning(
+            "SCIM: creating placeholder user for group member '%s'; matched no user by user_id, sso_user_id or "
+            "user_email. An SSO-provisioned user's real account stays teamless if this is a mismatch",
+            user_id,
+        )
+
     creations: Final = tuple(
         [
             (
@@ -2428,7 +2598,9 @@ async def _process_group_patch_operations(
             )
 
             if op_type == "remove":
-                final_members = final_members - {_member_value(member) for member in patched_members}
+                final_members = final_members - await _member_ids_to_drop(
+                    patched_members, frozenset(final_members), prisma_client
+                )
             else:
                 member_result = await _resolve_group_member_ids(
                     members=patched_members,

litellm/proxy/management_endpoints/team_endpoints.py

@@ -2640,51 +2640,63 @@ async def _process_team_members(
     return updated_users, updated_team_memberships
 
 
+def _resolve_member_identity(member: Member, updated_users: Sequence[LiteLLM_UserTable]) -> Member:
+    """Return ``member`` with whichever of ``user_id`` / ``user_email`` the caller left out filled in.
+
+    The roster entry is a snapshot, so whatever is missing here is missing for good.
+    Resolution runs both ways off the user rows the add just touched: added by email
+    -> stamp the user_id, added by user_id -> stamp the email. A value the caller
+    supplied is never overwritten.
+    """
+    resolved_user_id: Final = member.user_id or next(
+        (
+            user.user_id
+            for user in updated_users
+            if member.user_email is not None and user.user_email == member.user_email
+        ),
+        None,
+    )
+    resolved_user_email: Final = member.user_email or next(
+        (
+            user.user_email
+            for user in updated_users
+            if resolved_user_id is not None and user.user_id == resolved_user_id and user.user_email is not None
+        ),
+        None,
+    )
+    return member.model_copy(
+        update={  # mutable-ok: pydantic update payload
+            "user_id": resolved_user_id,
+            "user_email": resolved_user_email,
+        }
+    )
+
+
+def _member_already_in_team(member: Member, complete_team_data: LiteLLM_TeamTable) -> bool:
+    return any(
+        (member.user_id is not None and existing_member.user_id == member.user_id)
+        or (member.user_email is not None and existing_member.user_email == member.user_email)
+        for existing_member in complete_team_data.members_with_roles
+    )
+
+
 async def _update_team_members_list(
     data: TeamMemberAddRequest,
     complete_team_data: LiteLLM_TeamTable,
     updated_users: list[LiteLLM_UserTable],
 ) -> None:
     """Update the team's members_with_roles list."""
-    if isinstance(data.member, Member):
-        new_member: Final = data.member.model_copy()
+    requested_members: Final[Sequence[Member]] = (
+        (data.member,) if isinstance(data.member, Member) else tuple(data.member)
+    )
+    resolved_members: Final = tuple(_resolve_member_identity(m, updated_users) for m in requested_members)
 
-        # get user id
-        if new_member.user_id is None and new_member.user_email is not None:
-            for user in updated_users:
-                if user.user_email is not None and user.user_email == new_member.user_email:
-                    new_member.user_id = user.user_id
-
-        # Check if member already exists in team before adding
-        member_already_exists = False
-        for existing_member in complete_team_data.members_with_roles:
-            if (new_member.user_id is not None and existing_member.user_id == new_member.user_id) or (
-                new_member.user_email is not None and existing_member.user_email == new_member.user_email
-            ):
-                member_already_exists = True
-                break
-
-        if not member_already_exists:
-            complete_team_data.members_with_roles.append(new_member)
-
-    elif isinstance(data.member, list):
-        for nm in data.member:
-            if nm.user_id is None and nm.user_email is not None:
-                for user in updated_users:
-                    if user.user_email is not None and user.user_email == nm.user_email:
-                        nm.user_id = user.user_id
-
-            # Check if member already exists in team before adding
-            member_already_exists = False
-            for existing_member in complete_team_data.members_with_roles:
-                if (nm.user_id is not None and existing_member.user_id == nm.user_id) or (
-                    nm.user_email is not None and existing_member.user_email == nm.user_email
-                ):
-                    member_already_exists = True
-                    break
-
-            if not member_already_exists:
-                complete_team_data.members_with_roles.append(nm)
+    # extend() consumes the generator as it appends, so a member already added by this
+    # same call is seen by the next _member_already_in_team check - the batch dedupes
+    # against itself exactly as the append-one-at-a-time loop this replaced did.
+    complete_team_data.members_with_roles.extend(  # rebind-ok: this helper's contract is to grow the caller's roster in place
+        m for m in resolved_members if not _member_already_in_team(m, complete_team_data)
+    )
 
 
 async def _add_team_members_to_team(
@@ -4086,6 +4098,39 @@ async def _add_team_member_budget_table(
     return team_info_response_object
 
 
+async def _hydrate_member_emails(
+    prisma_client: PrismaClient,
+    members: Sequence[Member],
+) -> tuple[Member, ...]:
+    """Fill in ``user_email`` for roster entries that were stored without one.
+
+    ``members_with_roles`` is a denormalized snapshot written at add-time, so an entry
+    stored with ``user_email=None`` keeps that null even once the user row has an email.
+    Look the missing ones up in ``LiteLLM_UserTable`` (one indexed query) and fill them
+    in. A stored email is never overwritten - the snapshot stays the source of truth
+    wherever it has a value.
+    """
+    missing_user_ids: Final = frozenset(m.user_id for m in members if not m.user_email and m.user_id is not None)
+    if not missing_user_ids:
+        return tuple(members)
+
+    user_rows: Final[Sequence[LiteLLM_UserTable]] = await _user_db(prisma_client).find_many(
+        where={  # mutable-ok: Prisma query filters are dict-shaped
+            "user_id": {  # mutable-ok: Prisma query filters are dict-shaped
+                "in": sorted(missing_user_ids)
+            }
+        }
+    )
+    email_by_user_id: Final = MappingProxyType({u.user_id: u.user_email for u in user_rows if u.user_email})
+
+    return tuple(
+        m.model_copy(update={"user_email": email_by_user_id[m.user_id]})  # mutable-ok: pydantic update payload
+        if not m.user_email and m.user_id in email_by_user_id
+        else m
+        for m in members
+    )
+
+
 async def _resolve_team_access_group_resources(
     _team_info: TeamInfoResponseObjectTeamTable,
 ) -> TeamInfoResponseObjectTeamTable:
@@ -4221,9 +4266,22 @@ async def team_info(
         # Resolve resources inherited from access groups
         resolved_team_info: Final = await _resolve_team_access_group_resources(_team_info)
 
+        # Fill in emails the add-time roster snapshot never captured
+        hydrated_members: Final = await _hydrate_member_emails(
+            prisma_client=prisma_client,
+            members=resolved_team_info.members_with_roles,
+        )
+        hydrated_team_info: Final = resolved_team_info.model_copy(
+            update={  # mutable-ok: pydantic update payload
+                # list(), not the tuple: model_copy skips validation, so the field has
+                # to be handed the list[Member] the response model declares.
+                "members_with_roles": list(hydrated_members)  # mutable-ok: declared list[Member]
+            }
+        )
+
         response_object: Final = TeamInfoResponseObject(
             team_id=team_id,
-            team_info=resolved_team_info,
+            team_info=hydrated_team_info,
             keys=keys,
             team_memberships=returned_tm,
         )

litellm/proxy/pass_through_endpoints/streaming_handler.py

@@ -106,7 +106,7 @@ class PassThroughStreamingHandler:
                     )  # rebind-ok: SSE frame reassembly buffer across transport chunks
                     if complete_frames:
                         yield ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection(
-                            complete_frames, resolved_model_name
+                            complete_frames, resolved_model_name, litellm_logging_obj
                         )
                 if pending:
                     yield pending

litellm/proxy/proxy_server.py

@@ -6840,6 +6840,20 @@ class ProxyConfig:
         if self._should_load_db_object(object_type="config_overrides"):
             await self._init_hashicorp_vault_config_override(prisma_client=prisma_client)
 
+        await self._apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client)
+
+    async def _apply_safe_litellm_settings_overrides_from_db(self, prisma_client: PrismaClient) -> None:
+        config_record: Final = await get_config_param(prisma_client, "litellm_settings")
+        if config_record is None or config_record.param_value is None:
+            return
+        raw_settings: Final = config_record.param_value
+        litellm_settings: Final = json.loads(raw_settings) if isinstance(raw_settings, str) else raw_settings
+        if not isinstance(litellm_settings, dict):
+            return
+        for key, value in litellm_settings.items():
+            if key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES:
+                setattr(litellm, key, value)
+
     async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient):
         """
         Initialize MCP semantic filter settings from database.
@@ -8875,6 +8889,7 @@ class ProxyStartupEvent:
                 proxy_logging_obj=proxy_logging_obj,
                 prisma_client=prisma_client,
                 reset_settings=get_budget_reset_settings(),
+                pod_lock_manager=proxy_logging_obj.db_spend_update_writer.pod_lock_manager,
             )
 
             scheduler.add_job(

litellm/proxy/public_endpoints/provider_create_fields.json

@@ -772,6 +772,34 @@
     ],
     "default_model_placeholder": "gpt-3.5-turbo"
   },
+  {
+    "provider": "Cognition",
+    "provider_display_name": "Cognition",
+    "litellm_provider": "cognition",
+    "credential_fields": [
+      {
+        "key": "api_base",
+        "label": "API Base",
+        "placeholder": "https://api.cognition.ai/v1",
+        "tooltip": null,
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "api_key",
+        "label": "API Key",
+        "placeholder": null,
+        "tooltip": null,
+        "required": true,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      }
+    ],
+    "default_model_placeholder": "cognition/swe-1.7"
+  },
   {
     "provider": "Cohere",
     "provider_display_name": "Cohere",

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -10,6 +10,7 @@ from pydantic import BaseModel
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import (
+    LITELLM_PROXY_MASTER_KEY_ALIAS,
     LITELLM_TRUNCATED_PAYLOAD_FIELD,
     LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
     REDACTED_BY_LITELM_STRING,
@@ -22,6 +23,7 @@ from litellm.litellm_core_utils.core_helpers import (
     reconstruct_model_name,
 )
 from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
+from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes
 from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
 from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
@@ -54,13 +56,6 @@ def _get_max_string_length_prompt_in_db() -> int:
         return DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB
 
 
-def _hash_api_key_for_spend_log(api_key: str) -> str:
-    stripped: Final = api_key[7:] if api_key[:7].lower() == "bearer " else api_key
-    if stripped.startswith("sk-"):
-        return hash_token(stripped)
-    return stripped
-
-
 def _is_master_key(api_key: str | None, _master_key: str | None) -> bool:
     """
     Raw-only constant-time master-key comparison. The hashed form is never
@@ -71,6 +66,28 @@ def _is_master_key(api_key: str | None, _master_key: str | None) -> bool:
     return secrets.compare_digest(api_key, _master_key)
 
 
+_HASHED_JWT_RE = re.compile(r"hashed-jwt-[a-fA-F0-9]{64}")
+
+
+def _is_non_secret_key_value(value: str) -> bool:
+    return (
+        value == LITELLM_PROXY_MASTER_KEY_ALIAS
+        or is_valid_sha256_hash(value)
+        or _HASHED_JWT_RE.fullmatch(value) is not None
+    )
+
+
+def _redact_logged_api_key(value: str | None, *, already_redacted: bool = False) -> str | None:
+    if not isinstance(value, str) or not value:
+        return None
+    stripped: Final = re.sub(r"(?i)^bearer ", "", value)
+    if not stripped:
+        return None
+    if already_redacted and _is_non_secret_key_value(stripped):
+        return stripped
+    return hash_token(stripped)
+
+
 def _get_spend_logs_metadata(
     metadata: dict | None,
     applied_guardrails: list[str] | None = None,
@@ -124,9 +141,12 @@ def _get_spend_logs_metadata(
 
     # Filter the metadata dictionary to include only the specified keys
     clean_metadata: Final = SpendLogsMetadata(**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__})
-    raw_user_api_key: Final = clean_metadata.get("user_api_key")
-    if raw_user_api_key is not None and isinstance(raw_user_api_key, str):
-        clean_metadata["user_api_key"] = _hash_api_key_for_spend_log(raw_user_api_key)
+    _raw_key: Final = clean_metadata.get("user_api_key")
+    _trusted_hash: Final = metadata.get("user_api_key_hash")
+    _already_redacted: Final = (
+        isinstance(_trusted_hash, str) and _is_non_secret_key_value(_trusted_hash) and _trusted_hash == _raw_key
+    )
+    clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_redacted=_already_redacted)
     clean_metadata["applied_guardrails"] = applied_guardrails
     clean_metadata["batch_models"] = batch_models
     clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata
@@ -282,16 +302,23 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
         standard_logging_prompt_tokens = standard_logging_payload.get("prompt_tokens", 0)
         standard_logging_completion_tokens = standard_logging_payload.get("completion_tokens", 0)
         standard_logging_total_tokens = standard_logging_payload.get("total_tokens", 0)
-    if api_key is not None and isinstance(api_key, str):
-        api_key = _hash_api_key_for_spend_log(api_key)
+    _trusted_hash = metadata.get("user_api_key_hash")
+    _key_already_redacted = (
+        isinstance(_trusted_hash, str) and _is_non_secret_key_value(_trusted_hash) and _trusted_hash == api_key
+    )
+    api_key = _redact_logged_api_key(api_key, already_redacted=_key_already_redacted) or ""
 
     if (
         standard_logging_payload is not None
     ):  # [TODO] migrate completely to sl payload. currently missing pass-through endpoint data
-        api_key = api_key or standard_logging_payload["metadata"].get("user_api_key_hash") or ""
+        api_key = (
+            api_key
+            or _redact_logged_api_key(
+                standard_logging_payload["metadata"].get("user_api_key_hash"), already_redacted=True
+            )
+            or ""
+        )
         end_user_id = end_user_id or standard_logging_payload["metadata"].get("user_api_key_end_user_id")
-    # BUG FIX: Don't overwrite api_key when standard_logging_payload is None
-    # The api_key was already extracted from metadata (line 243) and hashed (lines 256-259)
     request_tags = safe_dumps(metadata.get("tags", [])) if isinstance(metadata.get("tags", []), list) else "[]"
     if (
         standard_logging_payload is not None and standard_logging_payload.get("request_tags") is not None

litellm/types/utils.py

@@ -11,6 +11,7 @@ from typing import (
     get_args,
 )
 
+import httpx
 from openai._models import BaseModel as OpenAIObject
 from openai.types.audio.transcription_create_params import (
     FileTypes as FileTypes,
@@ -49,7 +50,7 @@ from litellm.types.llms.base import (
 )
 from litellm.types.mcp import MCPServerCostInfo
 
-from ..litellm_core_utils.core_helpers import map_finish_reason
+from ..litellm_core_utils.core_helpers import map_finish_reason, process_response_headers
 from .agents import LiteLLMSendMessageResponse
 from .guardrails import GuardrailEventHooks
 from .llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
@@ -1916,6 +1917,10 @@ class ModelResponseBase(OpenAIObject):
 
     _response_headers: dict | None = None
 
+    def set_provider_response_headers(self, headers: httpx.Headers) -> None:
+        """Surface a provider's raw response headers to the caller as `llm_provider-*` headers."""
+        self._hidden_params["additional_headers"] = process_response_headers(headers)
+
     def model_dump(self, **kwargs):
         """Default to exclude_unset to avoid Pydantic serializer warnings for OpenAIObject-derived types."""
         if "exclude_unset" not in kwargs and "exclude_none" not in kwargs:
@@ -3788,6 +3793,7 @@ class LlmProviders(str, Enum):
     TENSORMESH = "tensormesh"
     LIBERTAI = "libertai"
     PINSTRIPES = "pinstripes"
+    COGNITION = "cognition"
     DARKBLOOM = "darkbloom"
     META = "meta"
     LITELLM_AGENT = "litellm_agent"

provider_endpoints_support.json

@@ -563,6 +563,23 @@
         "interactions": true
       }
     },
+    "cognition": {
+      "display_name": "Cognition (`cognition`)",
+      "url": "https://docs.litellm.ai/docs/providers/cognition",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": true,
+        "responses": true,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false
+      }
+    },
     "cohere": {
       "display_name": "Cohere (`cohere`)",
       "url": "https://docs.litellm.ai/docs/providers/cohere",

ruff-tests.toml

@@ -14,6 +14,18 @@
 # B018    a bare attribute access or literal, usually a call missing its parens
 # PLW0127 `x = x` self-assignment, dead code that reads like a narrowing or a fixup
 # PLR0133 comparison of two constants, e.g. `assert True == True`
+# B017    `pytest.raises(Exception)` accepts the TypeError a refactor introduced just as
+#         readily as the rejection under test, so a crash reads as a pass. Narrow to the
+#         real type, or add `match=` where the code genuinely raises a bare Exception
+# PT012   a `pytest.raises` block that runs on past the raising call. Everything after
+#         that call is dead, so an `assert` sitting there is never checked. Keep the
+#         block to the call itself and put the assertions below it
+# PT011   `pytest.raises(Exception)` / `(ValueError)` / `(OSError)` with no `match=`. The
+#         block passes on any error that broad, so the TypeError a refactor introduced
+#         reads as the rejection under test. Pin the message the code actually raises
+# PT014   the same `parametrize` case listed twice. The copy re-runs an assertion that
+#         already passed and adds no coverage, and it usually marks a case someone meant
+#         to vary and forgot to edit
 #
 # No target-version here on purpose: it resolves from requires-python (>=3.10), so
 # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that
@@ -21,4 +33,4 @@
 
 line-length = 120
 
-lint.select = ["F821", "B011", "B015", "B018", "PT015", "PLR0133", "PLW0127"]
+lint.select = ["F821", "B011", "B015", "B017", "B018", "PT011", "PT012", "PT014", "PT015", "PLR0133", "PLW0127"]

test-quality-budget.json

@@ -9,7 +9,7 @@
     "limit": 1078
   },
   "TQ004": {
-    "limit": 770
+    "limit": 768
   },
   "TQ005": {
     "limit": 2832

tests/e2e/batches/batch_client.py

@@ -40,8 +40,15 @@ class FileObject(BaseModel):
 
 
 class FileList(BaseModel):
+    """GET /v1/files page. The cursors are modelled because they are part of the
+    page's isolation contract: they must address rows in `data`, never rows the
+    caller was not allowed to see."""
+
     object: str | None = None
     data: list[FileObject] = []
+    first_id: str | None = None
+    last_id: str | None = None
+    has_more: bool | None = None
 
 
 class BatchObject(BaseModel):

tests/e2e/batches/test_batches_e2e.py

@@ -572,6 +572,41 @@ class TestOpenAIFiles:
             f"listed file must round-trip the upload purpose, got {match.purpose!r}"
         )
 
+    @pytest.mark.covers(
+        "llm.files.openai.list_isolation.nonstream.works",
+        exercised_on=["files"],
+    )
+    def test_list_page_cursors_address_only_the_callers_own_files(
+        self, client: BatchClient, resources: ResourceManager
+    ) -> None:
+        """Pins GitHub issue #36087: a list page's pagination cursors must address
+        rows in that page.
+
+        The proxy fronts one shared provider account, so the upstream page is the
+        whole organization's. The gateway narrows `data` to the files the caller
+        owns, and `first_id` / `last_id` have to be narrowed with it: left as the
+        upstream org's, they hand any caller raw provider file ids belonging to
+        other tenants, which is the handle the file routes accept.
+        """
+        key = resources.key(user_id=f"e2e-file-list-{unique_marker()}")
+
+        listed = unwrap(client.list_files(key=key))
+
+        expected_first = listed.data[0].id if listed.data else None
+        expected_last = listed.data[-1].id if listed.data else None
+        assert listed.first_id == expected_first, (
+            f"first_id {listed.first_id!r} is not the first row this caller can see "
+            f"({expected_first!r}); the page leaked another caller's file id"
+        )
+        assert listed.last_id == expected_last, (
+            f"last_id {listed.last_id!r} is not the last row this caller can see "
+            f"({expected_last!r}); the page leaked another caller's file id"
+        )
+        assert listed.has_more is not True, (
+            "the page advertises another page, but the proxy never forwards a cursor "
+            "upstream, so following it re-serves this same page forever"
+        )
+
     @pytest.mark.covers(
         "llm.files.openai.retrieve.nonstream.works",
         exercised_on=["files"],

tests/e2e/coverage_registry/llm_conversational.yaml

@@ -63,6 +63,7 @@
 - {id: llm.responses.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.9 / LIT-4778", rationale: "Responses missing/empty input and missing model are rejected"}
 - {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"}
 - {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"}
+- {id: llm.responses.openai.passthrough.stream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "A streamed POST /openai_passthrough/v1/responses is costed and keyed by the provider response id; it used to log a zero-cost row under a random id (GitHub issue #36523)"}
 - {id: llm.responses.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Tool calls via Responses API"}
 - {id: llm.responses.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Responses API"}
 - {id: llm.responses.anthropic.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Anthropic translation (smoke)"}

tests/e2e/coverage_registry/llm_nonconversational.yaml

@@ -3,6 +3,7 @@
 - {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"}
 - {id: llm.embeddings.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.3 / LIT-4778", rationale: "Missing model/input on /embeddings return client errors"}
 - {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"}
+- {id: llm.embeddings.openai.passthrough.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "POST /openai_passthrough/v1/embeddings is costed; the route wrote no spend row at all, so budgets never saw traffic OpenAI was billing for (GitHub issue #36646)"}
 - {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"}
 - {id: llm.embeddings.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/embed/embedding.py", rationale: "Bedrock Titan embeddings"}
 - {id: llm.embeddings.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_embeddings/embedding_handler.py", rationale: "Vertex embeddings"}
@@ -13,6 +14,7 @@
 - {id: llm.batches.openai.cancel.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Batch cancel"}
 - {id: llm.batches.openai.list.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Batch list envelope"}
 - {id: llm.batches.openai.file_lifecycle.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "File upload/retrieve/delete for batch flow"}
+- {id: llm.batches.openai.passthrough.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_passthrough_e2e.py", rationale: "GET /openai_passthrough/v1/batches relays OpenAI's own batch page; the dedicated prefix must not bind as a provider name on the /{provider}/v1/batches route (GitHub issue #36086)"}
 - {id: llm.batches.openai_encoded.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py", rationale: "Encoded scenario lifecycle"}
 - {id: llm.batches.openai_unified.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py", rationale: "Unified/managed-id scenario"}
 - {id: llm.batches.openai_model_param.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py", rationale: "Model-param scenario"}
@@ -29,6 +31,8 @@
 - {id: llm.files.openai.retrieve.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "files_endpoints.py", rationale: "File retrieve by id"}
 - {id: llm.files.openai.delete.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "files_endpoints.py", rationale: "File delete returns deleted=true"}
 - {id: llm.files.openai.list.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "files_endpoints.py", rationale: "File list paginated"}
+- {id: llm.files.openai.list_isolation.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "GET /v1/files pagination cursors address only rows the caller owns; on a shared provider account the upstream cursors otherwise hand out other tenants' raw provider file ids (GitHub issue #36087)"}
+- {id: llm.files.openai.passthrough.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_passthrough_e2e.py", rationale: "POST/DELETE /openai_passthrough/v1/files relay OpenAI's own file object; the dedicated prefix must not bind as a provider name on the /{provider}/v1/files route (GitHub issue #36086)"}
 - {id: llm.files.azure_openai.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:45", rationale: "Azure file upload managed backend"}
 - {id: llm.files.vertex.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:52", rationale: "Vertex file upload to GCS"}
 - {id: llm.files.bedrock.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:59", rationale: "Bedrock file upload to S3"}

tests/e2e/lifecycle.py

@@ -52,7 +52,7 @@ class ResourceManager:
     """
 
     client: ResourceClient
-    _cleanups: List[Callable[[], None]] = field(
+    _cleanups: List[Callable[[], object]] = field(
         default_factory=list
     )  # mutable-ok: append-only teardown registry
 
@@ -60,8 +60,11 @@ class ResourceManager:
         """No global setup needed today; present for lifecycle symmetry."""
         return None
 
-    def defer(self, cleanup: Callable[[], None]) -> None:
-        """Register a teardown action for any resource the test just created."""
+    def defer(self, cleanup: Callable[[], object]) -> None:
+        """Register a teardown action for any resource the test just created.
+
+        Whatever the action returns is discarded, so a delete that answers with a
+        response model can be deferred directly."""
         self._cleanups.append(cleanup)
 
     def key(self, models: list[str] | None = None, user_id: str | None = "e2e-test-user") -> str:

tests/e2e/llm_translation/passthrough_client.py

@@ -15,7 +15,7 @@ from dataclasses import dataclass
 from pydantic import BaseModel, Field
 
 from proxy_client import ProxyClient
-from e2e_http import Headers, StreamingResponse
+from e2e_http import FileUploadForm, Headers, NoBody, Result, StreamingResponse
 from models import ChatMessage
 
 
@@ -113,6 +113,76 @@ class OpenAIChatBody(BaseModel):
     max_completion_tokens: int = 64
 
 
+class PassthroughFileObject(BaseModel):
+    id: str
+    object: str | None = None
+    purpose: str | None = None
+    filename: str | None = None
+    bytes: int | None = None
+
+
+class PassthroughFileDeleted(BaseModel):
+    id: str
+    deleted: bool
+
+
+class PassthroughListEntry(BaseModel):
+    id: str
+
+
+class ResponsesUsage(BaseModel):
+    input_tokens: int
+    output_tokens: int
+
+
+class ResponsesObject(BaseModel):
+    id: str
+    usage: ResponsesUsage | None = None
+
+
+class ResponsesStreamEvent(BaseModel):
+    """One SSE frame of a native Responses stream. Only the terminal frames carry a
+    `response`, so it stays optional and the deltas validate as themselves."""
+
+    type: str
+    response: ResponsesObject | None = None
+
+
+def completed_responses_object(result: StreamingResponse) -> ResponsesObject | None:
+    """The `response.completed` frame's response object, or None if the stream never
+    completed. Its `id` is what the spend row is keyed by on this route, and its
+    usage is what the row is priced from."""
+    events = (
+        ResponsesStreamEvent.model_validate_json(payload)
+        for payload in result.stream_events
+    )
+    completed = tuple(
+        event.response
+        for event in events
+        if event.type == "response.completed" and event.response is not None
+    )
+    return completed[-1] if completed else None
+
+
+class OpenAIResponsesBody(BaseModel):
+    model: str
+    input: str
+    stream: bool = False
+
+
+class OpenAIEmbeddingBody(BaseModel):
+    model: str
+    input: str
+
+
+class PassthroughBatchList(BaseModel):
+    """OpenAI's own batch page, relayed verbatim. `object` is required so a body
+    that is not an OpenAI list fails validation instead of passing vacuously."""
+
+    object: str
+    data: list[PassthroughListEntry]
+
+
 def _tags_header(tags: list[str] | None) -> str | None:
     return ",".join(tags) if tags else None
 
@@ -196,6 +266,66 @@ class PassthroughClient:
             stream=stream,
         )
 
+    # ---- OpenAI file/batch routes under /openai_passthrough -------------
+    #
+    # Relayed to OpenAI untouched, which is the whole point of the prefix: the
+    # customer opts out of the gateway's managed-file handling here.
+
+    def openai_passthrough_upload_file(
+        self, key: str, *, content: bytes, filename: str
+    ) -> Result[PassthroughFileObject]:
+        return self.proxy.transport.upload(
+            "/openai_passthrough/v1/files",
+            headers=self.proxy.transport.bearer(key),
+            form=FileUploadForm(purpose="batch"),
+            filename=filename,
+            content=content,
+            response_type=PassthroughFileObject,
+        )
+
+    def openai_passthrough_delete_file(
+        self, key: str, file_id: str
+    ) -> Result[PassthroughFileDeleted]:
+        return self.proxy.transport.delete(
+            f"/openai_passthrough/v1/files/{file_id}",
+            headers=self.proxy.transport.bearer(key),
+            json=NoBody(),
+            response_type=PassthroughFileDeleted,
+        )
+
+    def openai_passthrough_list_batches(self, key: str) -> Result[PassthroughBatchList]:
+        return self.proxy.transport.get(
+            "/openai_passthrough/v1/batches",
+            headers=self.proxy.transport.bearer(key),
+            params=NoBody(),
+            response_type=PassthroughBatchList,
+        )
+
+    # ---- OpenAI inference routes under /openai_passthrough -------------
+    #
+    # Relayed to OpenAI verbatim, but still costed by the gateway: the customer
+    # budgets against this traffic, so a 200 that logs no spend is money the
+    # gateway never sees.
+
+    def openai_passthrough_responses(
+        self, key: str, model: str, text: str, *, stream: bool = False
+    ) -> StreamingResponse:
+        return self.proxy.transport.send(
+            "/openai_passthrough/v1/responses",
+            headers=self.proxy.transport.bearer(key),
+            json=OpenAIResponsesBody(model=model, input=text, stream=stream),
+            stream=stream,
+        )
+
+    def openai_passthrough_embed(
+        self, key: str, model: str, text: str
+    ) -> StreamingResponse:
+        return self.proxy.transport.send(
+            "/openai_passthrough/v1/embeddings",
+            headers=self.proxy.transport.bearer(key),
+            json=OpenAIEmbeddingBody(model=model, input=text),
+        )
+
     def openai_chat(
         self, key: str, model: str, text: str, *, max_completion_tokens: int = 64
     ) -> StreamingResponse:

tests/e2e/llm_translation/test_passthrough_e2e.py

@@ -13,8 +13,8 @@ A passthrough call returning non-2xx fails hard (never a skip); once it returns
 
 import pytest
 
-from e2e_config import unique_marker
-from e2e_http import StreamingResponse, require_successful_call
+from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
+from e2e_http import StreamingResponse, require_successful_call, unwrap
 from lifecycle import ResourceManager
 from models import KeyGenerateBody, SpendLogRow
 from passthrough_client import (
@@ -24,8 +24,11 @@ from passthrough_client import (
     JsonSchema,
     JsonSchemaProperty,
     PassthroughClient,
+    completed_responses_object,
 )
 
+EMBEDDING_MODEL = "text-embedding-3-small"
+
 pytestmark = pytest.mark.e2e
 
 
@@ -210,3 +213,129 @@ class TestPassthroughModelAllowlist:
             "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
             f"got {result.status_code}: {result.body[:300]}"
         )
+
+
+class TestOpenAIPassthroughPrefix:
+    """The dedicated `/openai_passthrough` prefix must reach OpenAI, not be
+    swallowed by the provider-scoped `/{provider}/v1/...` routes.
+
+    The customer fronts OpenAI's own file and batch APIs through this prefix
+    precisely to opt out of the gateway's managed-file handling. `/v1/files` and
+    `/v1/batches` also answer `/{provider}/v1/files` and `/{provider}/v1/batches`,
+    so `openai_passthrough` used to bind as a provider name and the request died
+    inside the gateway with a provider-lookup error, never reaching OpenAI.
+    """
+
+    @pytest.mark.covers("llm.files.openai.passthrough.nonstream.works")
+    def test_passthrough_prefix_uploads_a_file_to_openai(
+        self, client: PassthroughClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        """Pins GitHub issue #36086: a file upload through the dedicated prefix
+        reaches OpenAI's file API instead of 500ing on a provider-name lookup."""
+        content = f'{{"marker":"{unique_marker()}"}}\n'.encode()
+        uploaded = unwrap(
+            client.openai_passthrough_upload_file(
+                scoped_key, content=content, filename="e2e-passthrough-batch.jsonl"
+            )
+        )
+        resources.defer(
+            lambda: client.openai_passthrough_delete_file(scoped_key, uploaded.id)
+        )
+
+        assert uploaded.object == "file", (
+            f"/openai_passthrough/v1/files did not relay OpenAI's file object: {uploaded}"
+        )
+        assert uploaded.purpose == "batch"
+        assert uploaded.bytes == len(content)
+
+    @pytest.mark.covers("llm.batches.openai.passthrough.nonstream.works")
+    def test_passthrough_prefix_lists_batches_from_openai(
+        self, client: PassthroughClient, scoped_key: str
+    ) -> None:
+        """Pins GitHub issue #36086 on the batches route: the dedicated prefix
+        relays OpenAI's own batch page instead of dying on the provider lookup."""
+        listed = unwrap(client.openai_passthrough_list_batches(scoped_key))
+
+        assert listed.object == "list", (
+            f"/openai_passthrough/v1/batches did not relay OpenAI's batch page: {listed}"
+        )
+
+
+class TestOpenAIPassthroughSpend:
+    """A call relayed to OpenAI's own endpoints must still be costed.
+
+    The customer routes native OpenAI traffic through `/openai_passthrough` and
+    budgets against it, so a call that returns 200 while logging no spend is money
+    the gateway never sees and a budget that never trips. Streamed Responses calls
+    and embeddings each used to land exactly that way, on separate code paths.
+    """
+
+    @pytest.mark.covers("llm.responses.openai.passthrough.stream.cost_logged")
+    def test_streamed_responses_call_logs_its_cost(
+        self, client: PassthroughClient, scoped_key: str
+    ) -> None:
+        """Pins GitHub issue #36523: a streamed passthrough Responses call is billed
+        under the provider id the caller was served, never a $0 row under a random
+        id."""
+        result = client.openai_passthrough_responses(
+            scoped_key,
+            CHEAP_OPENAI_MODEL,
+            f"Say hi in one word. {unique_marker()}",
+            stream=True,
+        )
+        require_successful_call(result)
+        assert result.chunks > 0, "streamed responses passthrough produced no events"
+
+        completed = completed_responses_object(result)
+        assert completed is not None, (
+            f"the stream never delivered a response.completed frame, so there is no "
+            f"provider id to reconcile against: last events {result.stream_events[-3:]}"
+        )
+        assert completed.usage is not None, (
+            f"the completed response carried no usage to price from: {completed}"
+        )
+
+        rows = client.proxy.poll_logs_for_request_id(
+            completed.id, predicate=lambda rows: (rows[0].spend or 0) > 0
+        )
+        assert rows, (
+            f"no spend row for the response the customer was served ({completed.id}); "
+            "a streamed passthrough call OpenAI bills them for is invisible to the "
+            "gateway's own spend and budgets"
+        )
+        row = rows[0]
+        assert (row.spend or 0) > 0, f"streamed responses passthrough was not costed: {row}"
+        assert row.prompt_tokens == completed.usage.input_tokens, (
+            f"logged {row.prompt_tokens} prompt tokens, the response the customer read "
+            f"reported {completed.usage.input_tokens}"
+        )
+        assert row.completion_tokens == completed.usage.output_tokens, (
+            f"logged {row.completion_tokens} completion tokens, the response the customer "
+            f"read reported {completed.usage.output_tokens}"
+        )
+
+    @pytest.mark.covers("llm.embeddings.openai.passthrough.nonstream.cost_logged")
+    def test_embeddings_call_logs_its_cost(
+        self, client: PassthroughClient, scoped_key: str
+    ) -> None:
+        """Pins GitHub issue #36646: a passthrough embeddings call writes a priced
+        spend row instead of no row at all."""
+        result = client.openai_passthrough_embed(
+            scoped_key, EMBEDDING_MODEL, f"cost this sentence {unique_marker()}"
+        )
+        require_successful_call(result)
+        assert result.call_id, "embeddings passthrough returned no x-litellm-call-id"
+
+        rows = client.proxy.poll_logs_for_request_id(
+            result.call_id, predicate=lambda rows: (rows[0].spend or 0) > 0
+        )
+        assert rows, (
+            f"no spend row for embeddings call {result.call_id}; the customer is billed "
+            "by OpenAI for tokens the gateway never counted against their budget"
+        )
+        row = rows[0]
+        assert (row.spend or 0) > 0, f"embeddings passthrough was not costed: {row}"
+        assert (row.prompt_tokens or 0) > 0, (
+            f"the embeddings row logged no prompt tokens, so whatever cost it carries "
+            f"was not computed from the real usage: {row}"
+        )

tests/enterprise/litellm_enterprise/integrations/test_prometheus.py

@@ -400,7 +400,7 @@ def test_invalid_metric_name_validation():
     litellm.prometheus_metrics_config = test_config
 
     # Creating PrometheusLogger should raise ValueError due to invalid metric
-    with pytest.raises(ValueError) as exc_info:
+    with pytest.raises(ValueError, match='Configuration validation failed') as exc_info:
         PrometheusLogger()
 
     # Verify error message contains information about invalid metric
@@ -429,7 +429,7 @@ def test_invalid_labels_validation():
     litellm.prometheus_metrics_config = test_config
 
     # Creating PrometheusLogger should raise ValueError due to invalid labels
-    with pytest.raises(ValueError) as exc_info:
+    with pytest.raises(ValueError, match='Configuration validation failed') as exc_info:
         PrometheusLogger()
 
     # Verify error message contains information about invalid labels
@@ -598,7 +598,7 @@ def test_invalid_exclude_metric_name_raises(reset_prometheus_exclude_settings):
     litellm.prometheus_exclude_labels = None
     litellm.prometheus_exclude_metrics = ["not_a_real_metric"]
 
-    with pytest.raises(ValueError) as exc_info:
+    with pytest.raises(ValueError, match='Prometheus exclude configuration validation failed') as exc_info:
         PrometheusLogger()
 
     assert "not_a_real_metric" in str(exc_info.value)
@@ -612,7 +612,7 @@ def test_invalid_exclude_label_name_raises(reset_prometheus_exclude_settings):
     litellm.prometheus_exclude_metrics = None
     litellm.prometheus_exclude_labels = ["not_a_real_label"]
 
-    with pytest.raises(ValueError) as exc_info:
+    with pytest.raises(ValueError, match='Prometheus exclude configuration validation failed') as exc_info:
         PrometheusLogger()
 
     assert "not_a_real_label" in str(exc_info.value)

tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py

@@ -141,7 +141,7 @@ async def test_bedrock_apply_guardrail_api_failure():
         mock_api_request.side_effect = Exception("API connection failed")
 
         # Test the apply_guardrail method should raise an exception
-        with pytest.raises(Exception) as exc_info:
+        with pytest.raises(Exception, match='Bedrock guardrail failed: API connection failed') as exc_info:
             await guardrail.apply_guardrail(
                 inputs={"texts": ["This is a test message"]},
                 request_data={},

tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py

@@ -1653,7 +1653,7 @@ async def test_afile_retrieve_raises_error_when_no_router_and_file_object_none()
 
     unified_file_id = "test-unified-file-id"
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='LiteLLM Managed File object with id=test-unified-file-id') as exc_info:
         await proxy_managed_files.afile_retrieve(
             file_id=unified_file_id,
             litellm_parent_otel_span=None,
@@ -1719,7 +1719,7 @@ async def test_afile_retrieve_raises_error_for_non_managed_file():
     # Mock get_unified_file_id to return None (file not found)
     proxy_managed_files.get_unified_file_id = AsyncMock(return_value=None)
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='LiteLLM Managed File object with id=non-existent-file-id') as exc_info:
         await proxy_managed_files.afile_retrieve(
             file_id="non-existent-file-id",
             litellm_parent_otel_span=None,
@@ -2027,7 +2027,7 @@ async def test_list_batches_from_managed_objects_table_provider_filter_raises_ex
     )
 
     # Filtering by provider should raise Exception
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match="Filtering by 'provider' is not supported when using managed") as exc_info:
         await proxy_managed_files.list_user_batches(
             user_api_key_dict=UserAPIKeyAuth(user_id="test-user"),
             limit=10,
@@ -2053,7 +2053,7 @@ async def test_list_batches_from_managed_objects_table_target_model_name_filter_
     )
 
     # Filtering by provider should raise Exception
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match="Filtering by 'target_model_names' is not supported when") as exc_info:
         await proxy_managed_files.list_user_batches(
             user_api_key_dict=UserAPIKeyAuth(user_id="test-user"),
             limit=10,

tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py

@@ -29,6 +29,7 @@ from litellm_enterprise.proxy.management_endpoints.project_endpoints import (
 from litellm.proxy.proxy_server import (
     LitellmUserRoles,
 )
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 
 verbose_proxy_logger.setLevel(level=logging.DEBUG)
@@ -447,7 +448,7 @@ def test_check_team_project_limits_models_not_in_team():
         models=["gpt-5.5", "claude-3"],  # claude-3 not in team
     )
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match="not in team's allowed models\\. Team allowed models") as exc_info:
         _check_team_project_limits(team_object=team, data=data)
 
     assert "claude-3" in str(exc_info.value.detail)
@@ -475,7 +476,7 @@ def test_check_team_project_limits_budget_exceeds_team():
         max_budget=150.0,  # exceeds team's 100.0
     )
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='Project max_budget') as exc_info:
         _check_team_project_limits(team_object=team, data=data)
 
     assert "exceeds team's max_budget" in str(exc_info.value.detail)
@@ -550,7 +551,7 @@ def test_check_team_project_limits_tpm_exceeds_team():
         tpm_limit=20000,  # exceeds team's 10000
     )
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='Project tpm_limit') as exc_info:
         _check_team_project_limits(team_object=team, data=data)
 
     assert "exceeds team's tpm_limit" in str(exc_info.value.detail)
@@ -576,7 +577,7 @@ def test_check_team_project_limits_negative_budget():
         max_budget=-10.0,
     )
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='max_budget cannot be negative\\. Received') as exc_info:
         _check_team_project_limits(team_object=team, data=data)
 
     assert "cannot be negative" in str(exc_info.value.detail)
@@ -603,7 +604,7 @@ def test_check_team_project_limits_soft_budget_gte_max():
         soft_budget=100.0,  # equal to max, should fail
     )
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='must be strictly lower than max_budget') as exc_info:
         _check_team_project_limits(team_object=team, data=data)
 
     assert "must be strictly lower" in str(exc_info.value.detail)
@@ -1039,3 +1040,70 @@ async def test_project_eviction_publishes_cross_worker_invalidation(monkeypatch)
         )
 
     mock_publish.assert_awaited_once_with(cache_key=f"project_id:{project_id}")
+
+
+def _project_update_mocks(monkeypatch, stored_metadata: dict) -> mock.MagicMock:
+    existing_row = mock.MagicMock(
+        team_id=None, budget_id=None, object_permission_id=None, metadata=stored_metadata
+    )
+    mock_prisma = mock.MagicMock()
+    mock_prisma.jsonify_object = lambda data: data
+    mock_prisma.db.litellm_projecttable.find_unique = mock.AsyncMock(return_value=existing_row)
+    mock_prisma.db.litellm_projecttable.update = mock.AsyncMock(return_value=mock.MagicMock())
+
+    monkeypatch.setattr(litellm.proxy.proxy_server, "premium_user", True)
+    monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", mock_prisma)
+    monkeypatch.setattr(litellm.proxy.proxy_server, "user_api_key_cache", UserApiKeyCache())
+    return mock_prisma
+
+
+async def _run_project_update(project_id: str, **fields) -> None:
+    await update_project(
+        data=UpdateProjectRequest(project_id=project_id, **fields),
+        http_request=Request(scope={"type": "http"}),
+        user_api_key_dict=UserAPIKeyAuth(
+            user_role=LitellmUserRoles.PROXY_ADMIN,
+            api_key="sk-1234",
+            user_id="1234",
+        ),
+    )
+
+
+def _written_project_data(mock_prisma: mock.MagicMock) -> dict:
+    return mock_prisma.db.litellm_projecttable.update.await_args.kwargs["data"]
+
+
+@pytest.mark.asyncio
+async def test_update_project_clears_model_itpm_limit_sent_as_an_empty_map(monkeypatch):
+    """
+    LIT-4693 regression: an omitted key means "leave this alone", so the only way to drop a
+    per-model input/output TPM quota is to send it as an explicitly empty map. The written
+    metadata must stop carrying the quota, otherwise the proxy keeps enforcing a limit the
+    operator has already removed in the UI.
+    """
+    project_id = f"project-{uuid.uuid4()}"
+    mock_prisma = _project_update_mocks(
+        monkeypatch,
+        {"owner": "platform", "model_itpm_limit": {"gpt-4": 60}, "model_otpm_limit": {"gpt-4": 40}},
+    )
+
+    await _run_project_update(project_id, model_itpm_limit={}, model_otpm_limit={})
+
+    written_metadata = _written_project_data(mock_prisma)["metadata"]
+    assert written_metadata["model_itpm_limit"] == {}
+    assert written_metadata["model_otpm_limit"] == {}
+
+
+@pytest.mark.asyncio
+async def test_update_project_leaves_metadata_untouched_when_no_limit_is_sent(monkeypatch):
+    """
+    The other half of the same contract: an update that says nothing about the limits must not
+    write metadata at all. That is what makes a dropped key silently preserve the old quota, so
+    the UI has to send the empty map instead of omitting it.
+    """
+    project_id = f"project-{uuid.uuid4()}"
+    mock_prisma = _project_update_mocks(monkeypatch, {"model_itpm_limit": {"gpt-4": 60}})
+
+    await _run_project_update(project_id, description="renamed only")
+
+    assert "metadata" not in _written_project_data(mock_prisma)

tests/guardrails_tests/test_bedrock_guardrails.py

@@ -197,6 +197,7 @@ async def test_bedrock_guardrails_block_responses_api():
 
 @pytest.mark.asyncio
 async def test_bedrock_guardrails_with_streaming():
+    from fastapi import HTTPException
     from litellm.proxy.utils import ProxyLogging
     from litellm.types.guardrails import GuardrailEventHooks
 
@@ -204,7 +205,7 @@ async def test_bedrock_guardrails_with_streaming():
     mock_user_api_key_cache = MagicMock(spec=DualCache)
     mock_user_api_key_dict = UserAPIKeyAuth()
 
-    with pytest.raises(Exception):  # Assert that this raises an exception
+    async def _stream_through_guardrail():
         proxy_logging_obj = ProxyLogging(
             user_api_key_cache=mock_user_api_key_cache,
             premium_user=True,
@@ -239,6 +240,9 @@ async def test_bedrock_guardrails_with_streaming():
         async for chunk in response:
             print(chunk)
 
+    with pytest.raises(HTTPException):
+        await _stream_through_guardrail()
+
 
 @pytest.mark.asyncio
 async def test_bedrock_guardrails_with_streaming_no_violation():
@@ -1501,7 +1505,7 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming():
         mock_post.return_value = mock_bedrock_response
 
         # Should raise exception during streaming processing
-        with pytest.raises(HTTPException):
+        async def _drain():
             result_generator = (
                 guardrail_default.async_post_call_streaming_iterator_hook(
                     user_api_key_dict=mock_user_api_key_dict,
@@ -1510,10 +1514,12 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming():
                 )
             )
 
-            # Try to consume the generator - should raise exception
             async for chunk in result_generator:
                 pass
 
+        with pytest.raises(HTTPException):
+            await _drain()
+
     # Test 2: disable_exception_on_block=True. Streaming can't raise up to the
     # endpoint handler (SSE headers already flushed), so the block is delivered
     # as a synthetic stream with finish_reason=content_filter and the block

tests/guardrails_tests/test_dynamoai_guardrails.py

@@ -61,7 +61,7 @@ async def test_dynamoai_blocks_content_with_block_action():
         guardrail.should_run_guardrail = MagicMock(return_value=True)
 
         # Test that the guardrail raises ValueError for blocked content
-        with pytest.raises(ValueError) as exc_info:
+        with pytest.raises(ValueError, match='violation\\(s\\) detected') as exc_info:
             await guardrail.async_pre_call_hook(
                 data=request_data,
                 user_api_key_dict=UserAPIKeyAuth(),

tests/guardrails_tests/test_eu_ai_act_article5.py

@@ -20,6 +20,7 @@ from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_fil
 from litellm.types.proxy.guardrails.guardrail_hooks.litellm_content_filter import (
     ContentFilterCategoryConfig,
 )
+from fastapi import HTTPException
 
 
 # Test cases: (sentence, expected_result, reason)
@@ -210,7 +211,7 @@ class TestEUAIActArticle5ConditionalMatching:
         # Apply guardrail
         if expected == "BLOCK":
             # Should raise an exception or return modified response indicating block
-            with pytest.raises(Exception) as exc_info:
+            with pytest.raises(Exception, match='Content blocked: eu_ai_act_article') as exc_info:
                 await content_filter_guardrail.apply_guardrail(
                     inputs={"texts": [sentence]},
                     request_data=request_data,
@@ -275,7 +276,7 @@ class TestEUAIActEdgeCases:
         for sentence in sentences:
             request_data = {"messages": [{"role": "user", "content": sentence}]}
 
-            with pytest.raises(Exception):
+            with pytest.raises(HTTPException):
                 await content_filter_guardrail.apply_guardrail(
                     inputs={"texts": [sentence]},
                     request_data=request_data,
@@ -289,7 +290,7 @@ class TestEUAIActEdgeCases:
         request_data = {"messages": [{"role": "user", "content": sentence}]}
 
         # Should block (contains multiple violations)
-        with pytest.raises(Exception):
+        with pytest.raises(HTTPException):
             await content_filter_guardrail.apply_guardrail(
                 inputs={"texts": [sentence]},
                 request_data=request_data,

tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py

@@ -19,6 +19,7 @@ from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_fil
 from litellm.types.proxy.guardrails.guardrail_hooks.litellm_content_filter import (
     ContentFilterCategoryConfig,
 )
+from fastapi import HTTPException
 
 
 @pytest.fixture
@@ -82,7 +83,7 @@ class TestEUAIActFrench3Scenarios:
         print(f"{'='*70}\n")
 
         # Should raise an exception (blocked)
-        with pytest.raises(Exception) as exc_info:
+        with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'concevoir \\+") as exc_info:
             await content_filter_guardrail.apply_guardrail(
                 inputs={"texts": [sentence]},
                 request_data=request_data,
@@ -122,7 +123,7 @@ class TestEUAIActFrench3Scenarios:
         print(f"{'='*70}\n")
 
         # Should raise an exception (blocked)
-        with pytest.raises(Exception) as exc_info:
+        with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'créer \\+") as exc_info:
             await content_filter_guardrail.apply_guardrail(
                 inputs={"texts": [sentence]},
                 request_data=request_data,
@@ -193,7 +194,7 @@ class TestEUAIActFrench3Scenarios:
         print(f"{'='*70}\n")
 
         # Should raise an exception (blocked by conditional matching)
-        with pytest.raises(Exception) as exc_info:
+        with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'développer \\+") as exc_info:
             await content_filter_guardrail.apply_guardrail(
                 inputs={"texts": [sentence]},
                 request_data=request_data,
@@ -228,7 +229,7 @@ class TestFrenchEdgeCases:
         request_data = {"messages": [{"role": "user", "content": sentence}]}
 
         # Should block (contains "build" and "système de crédit social")
-        with pytest.raises(Exception):
+        with pytest.raises(HTTPException):
             await content_filter_guardrail.apply_guardrail(
                 inputs={"texts": [sentence]},
                 request_data=request_data,
@@ -257,7 +258,7 @@ class TestFrenchEdgeCases:
         request_data = {"messages": [{"role": "user", "content": sentence}]}
 
         # Should block (case-insensitive)
-        with pytest.raises(Exception):
+        with pytest.raises(HTTPException):
             await content_filter_guardrail.apply_guardrail(
                 inputs={"texts": [sentence]},
                 request_data=request_data,
@@ -277,7 +278,7 @@ class TestFrenchEdgeCases:
         request_data = {"messages": [{"role": "user", "content": sentence}]}
 
         # Should still block (no exception bypass)
-        with pytest.raises(Exception) as exc_info:
+        with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'créer \\+ crédit") as exc_info:
             await content_filter_guardrail.apply_guardrail(
                 inputs={"texts": [sentence]},
                 request_data=request_data,

tests/guardrails_tests/test_semantic_guard.py

@@ -10,6 +10,7 @@ sys.path.insert(0, os.path.abspath("../.."))
 from unittest.mock import MagicMock
 
 import pytest
+from fastapi import HTTPException
 
 
 class TestRouteLoader:
@@ -307,7 +308,7 @@ class TestContentFilterSqlInjectionTemplate:
     @pytest.mark.asyncio
     async def test_sql_always_block(self, sql_injection_guardrail, sentence, reason):
         request_data = {"messages": [{"role": "user", "content": sentence}]}
-        with pytest.raises(Exception):
+        with pytest.raises(HTTPException):
             await sql_injection_guardrail.apply_guardrail(
                 inputs={"texts": [sentence]},
                 request_data=request_data,
@@ -343,7 +344,7 @@ class TestContentFilterSqlInjectionTemplate:
         self, sql_injection_guardrail, sentence, reason
     ):
         request_data = {"messages": [{"role": "user", "content": sentence}]}
-        with pytest.raises(Exception):
+        with pytest.raises(HTTPException):
             await sql_injection_guardrail.apply_guardrail(
                 inputs={"texts": [sentence]},
                 request_data=request_data,
@@ -552,7 +553,7 @@ class TestContentFilterPromptInjectionTemplate:
     @pytest.mark.asyncio
     async def test_always_block(self, content_filter_guardrail, sentence, reason):
         request_data = {"messages": [{"role": "user", "content": sentence}]}
-        with pytest.raises(Exception):
+        with pytest.raises(HTTPException):
             await content_filter_guardrail.apply_guardrail(
                 inputs={"texts": [sentence]},
                 request_data=request_data,

tests/guardrails_tests/test_sg_mas_ai_guardrails.py

@@ -55,7 +55,7 @@ def _make_guardrail(yaml_filename: str, category_name: str) -> ContentFilterGuar
 
 async def _expect_block(guardrail: ContentFilterGuardrail, sentence: str, reason: str):
     request_data = {"messages": [{"role": "user", "content": sentence}]}
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='Content blocked: sg_mas_') as exc_info:
         await guardrail.apply_guardrail(
             inputs={"texts": [sentence]},
             request_data=request_data,

tests/guardrails_tests/test_sg_pdpa_guardrails.py

@@ -62,7 +62,7 @@ def _make_guardrail(yaml_filename: str, category_name: str) -> ContentFilterGuar
 async def _expect_block(guardrail: ContentFilterGuardrail, sentence: str, reason: str):
     """Assert that the guardrail BLOCKS the sentence."""
     request_data = {"messages": [{"role": "user", "content": sentence}]}
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='Content blocked: sg_pdpa_') as exc_info:
         await guardrail.apply_guardrail(
             inputs={"texts": [sentence]},
             request_data=request_data,

tests/image_gen_tests/test_image_generation.py

@@ -444,7 +444,7 @@ async def test_azure_image_generation_request_body():
     ) as mock_post:
         mock_post.side_effect = Exception("test")
 
-        with pytest.raises(Exception):
+        with pytest.raises(litellm.APIConnectionError):
             await aimage_generation(
                 model="azure/gpt-image-1",
                 prompt="test prompt",

tests/litellm_utils_tests/test_hashicorp.py

@@ -432,7 +432,7 @@ def test_hashicorp_get_url_rejects_path_traversal(monkeypatch, malicious_secret_
     monkeypatch.setenv("HCP_VAULT_TOKEN", "test-token-for-get-url-only")
     manager = HashicorpSecretManager()
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match='Invalid secret_name'):
         manager.get_url(malicious_secret_name)
 
 

tests/litellm_utils_tests/test_proxy_budget_reset.py

@@ -748,8 +748,9 @@ async def test_service_logger_keys_failure():
     ) = proxy_logging_obj.service_logging_obj.async_service_failure_hook.call_args
     event_metadata = kwargs.get("event_metadata", {})
     assert event_metadata.get("num_keys_found") == len(keys)
-    keys_found_str = event_metadata.get("keys_found", "")
-    assert "key1" in keys_found_str
+    # the row payload is deliberately absent: serializing every found row on the
+    # event loop is what blocked auth on the sweeping pod
+    assert "keys_found" not in event_metadata
     # Success hook should not be called.
     proxy_logging_obj.service_logging_obj.async_service_success_hook.assert_not_called()
 
@@ -866,8 +867,7 @@ async def test_service_logger_users_failure():
     ) = proxy_logging_obj.service_logging_obj.async_service_failure_hook.call_args
     event_metadata = kwargs.get("event_metadata", {})
     assert event_metadata.get("num_users_found") == len(users)
-    users_found_str = event_metadata.get("users_found", "")
-    assert "user1" in users_found_str
+    assert "users_found" not in event_metadata
     proxy_logging_obj.service_logging_obj.async_service_success_hook.assert_not_called()
 
 
@@ -983,8 +983,7 @@ async def test_service_logger_teams_failure():
     ) = proxy_logging_obj.service_logging_obj.async_service_failure_hook.call_args
     event_metadata = kwargs.get("event_metadata", {})
     assert event_metadata.get("num_teams_found") == len(teams)
-    teams_found_str = event_metadata.get("teams_found", "")
-    assert "team1" in teams_found_str
+    assert "teams_found" not in event_metadata
     proxy_logging_obj.service_logging_obj.async_service_success_hook.assert_not_called()
 
 
@@ -1113,8 +1112,8 @@ async def test_service_logger_endusers_failure():
     event_metadata = kwargs.get("event_metadata", {})
     assert event_metadata.get("num_budgets_found") == len(budgets)
     assert event_metadata.get("num_endusers_found") == len(endusers)
-    endusers_found_str = event_metadata.get("endusers_found", "")
-    assert "user1" in endusers_found_str
+    assert "endusers_found" not in event_metadata
+    assert "budgets_found" not in event_metadata
     proxy_logging_obj.service_logging_obj.async_service_success_hook.assert_not_called()
 
 

tests/litellm_utils_tests/test_utils.py

@@ -1334,7 +1334,7 @@ def test_validate_chat_completion_user_messages(messages, expected_bool):
         validate_chat_completion_user_messages(messages=messages)
     else:
         ## Invalid message
-        with pytest.raises(Exception):
+        with pytest.raises(Exception, match="Invalid user message at index 0"):
             validate_chat_completion_user_messages(messages=messages)
 
 
@@ -1354,7 +1354,7 @@ def test_validate_chat_completion_tool_choice(tool_choice, expected_bool):
     if expected_bool:
         validate_chat_completion_tool_choice(tool_choice=tool_choice)
     else:
-        with pytest.raises(Exception):
+        with pytest.raises(Exception, match="Invalid tool choice"):
             validate_chat_completion_tool_choice(tool_choice=tool_choice)
 
 
@@ -2147,7 +2147,7 @@ def test_validate_user_messages_invalid_content_type():
 
     messages = [{"content": [{"type": "invalid_type", "text": "Hello"}]}]
 
-    with pytest.raises(Exception) as e:
+    with pytest.raises(Exception, match='Please ensure all messages are valid OpenAI chat completion') as e:
         validate_chat_completion_user_messages(messages)
 
     assert "Invalid message" in str(e)

tests/litellm_utils_tests/test_validate_tool_choice.py

@@ -37,27 +37,27 @@ def test_validate_tool_choice_cursor_format():
 def test_validate_tool_choice_invalid_dict():
     """Test that invalid dict formats raise exceptions."""
     # Missing both type and function
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='Invalid tool choice, tool_choice=\\{\\}\\. Please ensure') as exc_info:
         validate_chat_completion_tool_choice({})
     assert "Invalid tool choice" in str(exc_info.value)
 
     # Invalid type value
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\{'type': 'invalid'\\}\\.") as exc_info:
         validate_chat_completion_tool_choice({"type": "invalid"})
     assert "Invalid tool choice" in str(exc_info.value)
 
     # Has type but missing function when type is "function"
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\{'type': 'function'\\}\\.") as exc_info:
         validate_chat_completion_tool_choice({"type": "function"})
     assert "Invalid tool choice" in str(exc_info.value)
 
 
 def test_validate_tool_choice_invalid_type():
     """Test that invalid types raise exceptions."""
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match="<class 'int'>\\. Expecting str, or dict\\. Please ensure") as exc_info:
         validate_chat_completion_tool_choice(123)
     assert "Got=<class 'int'>" in str(exc_info.value)
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\[\\]\\. Got=<class 'list'>\\.") as exc_info:
         validate_chat_completion_tool_choice([])
     assert "Got=<class 'list'>" in str(exc_info.value)

tests/llm_responses_api_testing/base_responses_api.py

@@ -28,6 +28,7 @@ from openai.types.responses.response_create_params import (
     ResponseInputParam,
 )
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+import openai
 
 
 def validate_responses_api_response(response, final_chunk: bool = False):
@@ -700,12 +701,12 @@ class BaseResponsesAPITest(ABC):
         base_completion_call_args = self.get_base_completion_call_args()
 
         if sync_mode:
-            with pytest.raises(Exception):
+            with pytest.raises(openai.APIError):
                 litellm.cancel_responses(
                     response_id="invalid_response_id_12345", **base_completion_call_args
                 )
         else:
-            with pytest.raises(Exception):
+            with pytest.raises(openai.APIError):
                 await litellm.acancel_responses(
                     response_id="invalid_response_id_12345", **base_completion_call_args
                 )

tests/llm_responses_api_testing/test_openai_responses_api.py

@@ -1643,10 +1643,13 @@ async def test_openai_responses_api_token_limit_error():
         model="gpt-5-mini", input=oversized_text, stream=True
     )
 
-    with pytest.raises(litellm.APIError) as exc_info:
+    async def _drain():
         async for event in response:
             print(event)
 
+    with pytest.raises(litellm.APIError) as exc_info:
+        await _drain()
+
     assert exc_info.value.status_code == 400
     assert "exceeds the context window" in str(exc_info.value)
 

tests/llm_responses_api_testing/test_responses_hooks.py

@@ -295,7 +295,7 @@ async def test_responses_streaming_failure_triggers_failure_handlers():
         call_type=CallTypes.responses.value,
     )
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match="boom"):
         iterator._process_chunk('{"delta": "chunk"}')
 
     # allow failure callbacks to run

tests/llm_translation/test_azure_openai.py

@@ -650,7 +650,7 @@ def test_azure_openai_responses_bridge():
         mock_responses.assert_called_once()
         assert (
             mock_responses.call_args.kwargs["model"]
-            == "test-azure-computer-use-preview"
+            == "azure/test-azure-computer-use-preview"
         )
         assert mock_responses.call_args.kwargs["custom_llm_provider"] == "azure"
 

tests/llm_translation/test_bedrock_completion.py

@@ -1890,7 +1890,7 @@ def test_bedrock_completion_test_4(modify_params):
         ]
         assert transformed_messages == expected_messages
     else:
-        with pytest.raises(Exception) as e:
+        with pytest.raises(Exception, match=r"litellm\.modify_params") as e:
             litellm.completion(**data)
         assert "litellm.modify_params" in str(e.value)
 

tests/llm_translation/test_bedrock_moonshot.py

@@ -12,6 +12,7 @@ This test suite verifies:
 """
 
 from base_llm_unit_tests import BaseLLMChatTest
+import httpx
 import pytest
 import sys
 import os
@@ -208,14 +209,6 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest):
         endpoint with the messages body. Iteration of the stream itself is
         not exercised here — moonshot streaming delegates to the OpenAI
         parser and is covered by the OpenAI test suite.
-
-        Note: bedrock invoke streaming cannot be intercepted by patching
-        the caller-supplied client, because ``CustomStreamWrapper.fetch_sync_stream``
-        at streaming_handler.py invokes the stored ``make_call`` partial with
-        ``client=litellm.module_level_client``, which overrides any client the
-        caller passed. Patch ``make_sync_call`` at its import site in
-        ``base_invoke_transformation`` so we observe the exact kwargs the
-        partial was built with at stream-wrapper construction time.
         """
         from litellm.utils import CustomStreamWrapper
 
@@ -225,7 +218,7 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest):
             captured.update(kwargs)
             # Return an empty iterator so the stream wrapper's iteration
             # doesn't try to parse real bytes.
-            return iter([])
+            return iter([]), httpx.Headers()
 
         with patch(
             "litellm.llms.bedrock.chat.invoke_transformations."
@@ -246,11 +239,6 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest):
                 aws_region_name="us-west-2",
             )
             assert isinstance(response, CustomStreamWrapper)
-            # Trigger fetch_sync_stream → make_call(...) → fake_make_sync_call.
-            try:
-                next(iter(response))
-            except StopIteration:
-                pass
 
         assert captured, "make_sync_call was never invoked"
         assert captured["api_base"].endswith("/invoke-with-response-stream")

tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py

@@ -982,7 +982,7 @@ def test_convert_to_model_response_object_with_real_error():
         },
     }
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception) as exc_info:  # noqa: PT011  # message rides on .message, str() is empty
         convert_to_model_response_object(
             model_response_object=ModelResponse(),
             response_object=response_object,
@@ -1243,7 +1243,7 @@ def test_convert_to_model_response_object_with_error_code_only():
         },
     }
 
-    with pytest.raises(Exception):
+    with pytest.raises(Exception) as exc_info:  # noqa: B017, PT011  # bare Exception, empty message, so status_code is the assertion
         convert_to_model_response_object(
             model_response_object=ModelResponse(),
             response_object=response_object,
@@ -1255,6 +1255,8 @@ def test_convert_to_model_response_object_with_error_code_only():
             convert_tool_call_to_json_mode=False,
         )
 
+    assert exc_info.value.status_code == 500
+
 
 def test_model_prefix_preservation():
     """
@@ -1421,7 +1423,7 @@ def test_error_message_includes_function_args():
         "choices": [{"index": 0}],
     }
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='in convert_to_model_response_object') as exc_info:
         convert_to_model_response_object(
             model_response_object=ModelResponse(),
             response_object=response_object,
@@ -2473,14 +2475,14 @@ class TestConvertToModelResponseObjectCompletion:
         assert "reasoning_content" not in (message.provider_specific_fields or {})
 
     def test_response_none_raises(self):
-        with pytest.raises(Exception):
+        with pytest.raises(Exception, match="Invalid response object"):
             convert_to_model_response_object(
                 response_object=None,
                 model_response_object=ModelResponse(),
             )
 
     def test_model_response_none_raises(self):
-        with pytest.raises(Exception):
+        with pytest.raises(Exception, match="Invalid response object"):
             convert_to_model_response_object(
                 response_object={
                     "choices": [

tests/llm_translation/test_openai.py

@@ -1458,7 +1458,7 @@ def test_responses_gpt54_with_xhigh_reasoning():
         # Stop execution right after request generation to avoid external API calls.
         mock_responses.side_effect = RuntimeError("stop_after_request_build")
 
-        with pytest.raises(Exception):
+        with pytest.raises(litellm.APIConnectionError):
             litellm.completion(
                 model="openai/responses/gpt-5.4",
                 messages=[{"role": "user", "content": "What is 2+2?"}],

tests/llm_translation/test_prompt_factory.py

@@ -1288,7 +1288,8 @@ def test_just_system_message():
             model="anthropic.claude-3-sonnet-20240229-v1:0",
             llm_provider="bedrock",
         )
-        assert "bedrock requires at least one non-system message" in str(e.value)
+
+    assert "bedrock requires at least one non-system message" in str(e.value)
 
 
 def test_convert_generic_image_chunk_to_openai_image_obj():
@@ -1844,7 +1845,7 @@ def test_parse_tool_call_arguments_malformed_json():
         parse_tool_call_arguments,
     )
 
-    with pytest.raises(ValueError) as exc_info:
+    with pytest.raises(ValueError, match="Failed to parse tool call arguments for tool 'load_skill") as exc_info:
         parse_tool_call_arguments(
             '{"skill_name": "pptx',
             tool_name="load_skill",
@@ -1876,7 +1877,7 @@ def test_convert_to_anthropic_tool_invoke_malformed_json():
         }
     ]
 
-    with pytest.raises(ValueError) as exc_info:
+    with pytest.raises(ValueError, match="Failed to parse tool call arguments for tool 'bad_tool") as exc_info:
         convert_to_anthropic_tool_invoke(tool_calls)
 
     error_msg = str(exc_info.value)
@@ -2022,7 +2023,7 @@ def test_parse_tool_call_arguments_still_raises_for_unrepairable():
         parse_tool_call_arguments,
     )
 
-    with pytest.raises(ValueError) as exc_info:
+    with pytest.raises(ValueError, match="Failed to parse tool call arguments for tool 'test_tool") as exc_info:
         parse_tool_call_arguments(
             '{"key": "unterminated',
             tool_name="test_tool",

tests/llm_translation/test_together_ai.py

@@ -20,7 +20,7 @@ import pytest
 class TestTogetherAI(BaseLLMChatTest):
     def get_base_completion_call_args(self) -> dict:
         litellm.set_verbose = True
-        return {"model": "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo"}
+        return {"model": "together_ai/openai/gpt-oss-20b"}
 
     def test_tool_call_no_arguments(self, tool_call_no_arguments):
         """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""

tests/llm_translation/test_triton.py

@@ -45,7 +45,7 @@ def test_split_embedding_by_shape_fails_with_shape_value_error():
             "data": [1, 2, 3, 4, 5, 6],
         }
     ]
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match='Shape must be of length'):
         TritonEmbeddingConfig.split_embedding_by_shape(
             data[0]["data"], data[0]["shape"]
         )

tests/llm_translation/test_unit_test_bedrock_invoke.py

@@ -59,7 +59,7 @@ def test_transform_request_invalid_provider(bedrock_transformer):
     """Test request transformation with invalid provider"""
     messages = [{"role": "user", "content": "Hello"}]
 
-    with pytest.raises(Exception) as exc_info:
+    with pytest.raises(Exception, match='Bedrock Invoke HTTPX: Unknown provider=None') as exc_info:
         bedrock_transformer.transform_request(
             model="invalid.model",
             messages=messages,

tests/local_testing/test_aim_guardrails.py

@@ -101,26 +101,26 @@ async def test_block_callback(mode: str):
         ],
     }
 
-    with pytest.raises(ProxyException, match="Jailbreak detected") as exc_info:
-        with patch(
-            "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
-            return_value=Response(
-                json={
-                    "analysis_result": {
-                        "analysis_time_ms": 212,
-                        "policy_drill_down": {},
-                        "session_entities": [],
-                    },
-                    "required_action": {
-                        "action_type": "block_action",
-                        "detection_message": "Jailbreak detected",
-                        "policy_name": "blocking policy",
-                    },
+    with patch(
+        "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
+        return_value=Response(
+            json={
+                "analysis_result": {
+                    "analysis_time_ms": 212,
+                    "policy_drill_down": {},
+                    "session_entities": [],
                 },
-                status_code=200,
-                request=Request(method="POST", url="http://aim"),
-            ),
-        ):
+                "required_action": {
+                    "action_type": "block_action",
+                    "detection_message": "Jailbreak detected",
+                    "policy_name": "blocking policy",
+                },
+            },
+            status_code=200,
+            request=Request(method="POST", url="http://aim"),
+        ),
+    ):
+        async def _call_guardrail():
             if mode == "pre_call":
                 await aim_guardrail.async_pre_call_hook(
                     data=data,
@@ -135,6 +135,9 @@ async def test_block_callback(mode: str):
                     call_type="completion",
                 )
 
+        with pytest.raises(ProxyException, match="Jailbreak detected") as exc_info:
+            await _call_guardrail()
+
     exc = exc_info.value
     assert exc.code == "400"
     assert exc.type == "invalid_request_error"

tests/local_testing/test_auth_utils.py

@@ -264,10 +264,6 @@ def test_get_end_user_id_from_request_body_backwards_compatibility():
             ["gpt-3.5-turbo", "gpt-4o-mini-general-deployment"],
         ),
         ({"model": "gpt-3.5-turbo"}, "gpt-3.5-turbo"),
-        (
-            {"model": "gpt-3.5-turbo, gpt-4o-mini-general-deployment"},
-            ["gpt-3.5-turbo", "gpt-4o-mini-general-deployment"],
-        ),
     ],
 )
 def test_get_model_from_request(request_data, expected_model):

tests/local_testing/test_completion.py

@@ -67,7 +67,7 @@ def test_completion_custom_provider_model_name():
     try:
         litellm.cache = None
         response = completion(
-            model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
+            model="together_ai/openai/gpt-oss-20b",
             messages=messages,
             logger_fn=logger_fn,
         )
@@ -2817,7 +2817,7 @@ def test_customprompt_together_ai():
         print(litellm.success_callback)
         print(litellm._async_success_callback)
         response = completion(
-            model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
+            model="together_ai/openai/gpt-oss-20b",
             messages=messages,
             roles={
                 "system": {
@@ -3657,7 +3657,7 @@ def test_completion_together_ai_stream():
     messages = [{"content": user_message, "role": "user"}]
     try:
         response = completion(
-            model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
+            model="together_ai/openai/gpt-oss-20b",
             messages=messages,
             stream=True,
             max_tokens=5,

tests/local_testing/test_completion_cost.py

@@ -625,17 +625,9 @@ def test_vertex_ai_completion_cost():
     print("calculated_input_cost: {}".format(calculated_input_cost))
 
 
-@pytest.mark.skip(reason="new test - WIP, working on fixing this")
 def test_vertex_ai_medlm_completion_cost():
     """Test for medlm completion cost ."""
 
-    with pytest.raises(Exception) as e:
-        model = "vertex_ai/medlm-medium"
-        messages = [{"role": "user", "content": "Test MedLM completion cost."}]
-        predictive_cost = completion_cost(
-            model=model, messages=messages, custom_llm_provider="vertex_ai"
-        )
-
     os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
     litellm.model_cost = litellm.get_model_cost_map(url="")
 

tests/local_testing/test_exceptions.py

@@ -1417,7 +1417,7 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model):
     import litellm
 
     litellm.set_verbose = True
-    with pytest.raises(Exception) as exc_info:
+    async def _call_with_bad_role():
         if sync_mode:
             litellm.completion(
                 model=model,
@@ -1433,6 +1433,9 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model):
                 sync_stream=sync_mode,
             )
 
+    with pytest.raises(Exception, match='litellm\\.BadRequestError: OpenAIException - Invalid value') as exc_info:
+        await _call_with_bad_role()
+
     assert exc_info.value.code == "invalid_value"
     assert exc_info.value.param is not None
     assert exc_info.value.type == "invalid_request_error"

tests/local_testing/test_file_types.py

@@ -23,13 +23,13 @@ class TestFileConsts:
     def test_get_file_extension_from_mime_type(self):
         assert get_file_extension_from_mime_type("audio/aac") == "aac"
         assert get_file_extension_from_mime_type("application/pdf") == "pdf"
-        with pytest.raises(ValueError):
+        with pytest.raises(ValueError, match='Unknown extension for mime type: application'):
             get_file_extension_from_mime_type("application/unknown")
 
     def test_get_file_type_from_extension(self):
         assert get_file_type_from_extension("aac") == FileType.AAC
         assert get_file_type_from_extension("pdf") == FileType.PDF
-        with pytest.raises(ValueError):
+        with pytest.raises(ValueError, match='Unknown file type for extension: unknown'):
             get_file_type_from_extension("unknown")
 
     def test_get_file_extension_for_file_type(self):

tests/local_testing/test_function_calling.py

@@ -357,14 +357,13 @@ def test_parallel_function_call_anthropic_error_msg(
 
         if expect_unsupported_params_error:
             with pytest.raises(litellm.UnsupportedParamsError) as e:
-                second_response = litellm.completion(
+                litellm.completion(
                     model=model,
                     messages=messages,
                     temperature=0.2,
                     seed=22,
                     drop_params=True,
-                )  # get a new response from the model where it can see the function response
-                print("second response\n", second_response)
+                )
         else:
             second_response = litellm.completion(
                 model=model,

tests/local_testing/test_get_llm_provider.py

@@ -569,5 +569,5 @@ class TestClaudeModelPatternMatching:
         )
 
         set_fallback_generalizations([])
-        with pytest.raises(Exception):
+        with pytest.raises(litellm.BadRequestError):
             litellm.get_llm_provider(model="claude-opus-4-9")

tests/local_testing/test_get_model_info.py

@@ -134,7 +134,6 @@ def test_get_model_info_bedrock_region():
         "ft:gpt-3.5-turbo:my-org:custom_suffix:id",
         "ft:gpt-4-0613:my-org:custom_suffix:id",
         "ft:davinci-002:my-org:custom_suffix:id",
-        "ft:gpt-4-0613:my-org:custom_suffix:id",
         "ft:babbage-002:my-org:custom_suffix:id",
         "gpt-35-turbo",
         "ada",

tests/local_testing/test_mock_request.py

@@ -128,13 +128,12 @@ def test_router_mock_request_with_mock_timeout():
         ],
     )
     with pytest.raises(litellm.Timeout):
-        response = router.completion(
+        router.completion(
             model="gpt-3.5-turbo",
             messages=[{"role": "user", "content": "Hey, I'm a mock request"}],
             timeout=3,
             mock_timeout=True,
         )
-        print(response)
     end_time = time.time()
     assert end_time - start_time >= 3, f"Time taken: {end_time - start_time}"
 

tests/local_testing/test_router_budget_limiter.py

@@ -160,13 +160,11 @@ async def test_provider_budgets_e2e_test_expect_to_fail():
     await asyncio.sleep(2.5)
 
     for _ in range(3):
-        with pytest.raises(Exception) as exc_info:
-            response = await router.acompletion(
+        with pytest.raises(Exception, match="Exceeded budget for provider") as exc_info:
+            await router.acompletion(
                 messages=[{"role": "user", "content": "Hello, how are you?"}],
                 model="anthropic/claude-sonnet-4-5-20250929",
             )
-            print(response)
-            print("response.hidden_params", response._hidden_params)
 
         await asyncio.sleep(0.5)
         # Verify the error is related to budget exceeded
@@ -596,13 +594,11 @@ async def test_deployment_budgets_e2e_test_expect_to_fail():
     await asyncio.sleep(2.5)
 
     for _ in range(3):
-        with pytest.raises(Exception) as exc_info:
-            response = await router.acompletion(
+        with pytest.raises(Exception, match="Exceeded budget for deployment") as exc_info:
+            await router.acompletion(
                 messages=[{"role": "user", "content": "Hello, how are you?"}],
                 model="openai/gpt-4o-mini",
             )
-            print(response)
-            print("response.hidden_params", response._hidden_params)
 
         await asyncio.sleep(0.5)
         # Verify the error is related to budget exceeded
@@ -650,14 +646,12 @@ async def test_tag_budgets_e2e_test_expect_to_fail():
     await asyncio.sleep(2.5)
 
     for _ in range(3):
-        with pytest.raises(Exception) as exc_info:
-            response = await router.acompletion(
+        with pytest.raises(Exception, match=f"Exceeded budget for tag='{TAG_NAME}'") as exc_info:
+            await router.acompletion(
                 messages=[{"role": "user", "content": "Hello, how are you?"}],
                 model="openai/gpt-4o-mini",
                 metadata={"tags": [TAG_NAME]},
             )
-            print(response)
-            print("response.hidden_params", response._hidden_params)
 
         await asyncio.sleep(0.5)
         # Verify the error is related to budget exceeded

tests/local_testing/test_router_fallbacks.py

@@ -1416,7 +1416,7 @@ async def test_router_fallbacks_default_and_model_specific_fallbacks(sync_mode):
         default_fallbacks=["bad-model"],
     )
 
-    with pytest.raises(Exception) as exc_info:
+    async def _call_bad_model():
         if sync_mode:
             resp = router.completion(
                 model="bad-model",
@@ -1429,6 +1429,9 @@ async def test_router_fallbacks_default_and_model_specific_fallbacks(sync_mode):
                 model="bad-model",
                 messages=[{"role": "user", "content": "Hey, how's it going?"}],
             )
+
+    with pytest.raises(Exception, match='litellm\\.AuthenticationError: AuthenticationError') as exc_info:
+        await _call_bad_model()
     assert isinstance(
         exc_info.value, litellm.AuthenticationError
     ), f"Expected AuthenticationError, but got {type(exc_info.value).__name__}"

tests/local_testing/test_router_max_parallel_requests.py

@@ -205,9 +205,12 @@ async def test_max_parallel_requests_tpm_rate_limiting_base_case():
         num_retries=0,
     )
 
-    with pytest.raises(litellm.RateLimitError):
+    async def _exceed_limit():
         for _ in range(2):
             await router.acompletion(
                 model="gpt-4o-2024-08-06",
                 messages=_messages,
             )
+
+    with pytest.raises(litellm.RateLimitError):
+        await _exceed_limit()

tests/local_testing/test_streaming.py

@@ -2926,11 +2926,14 @@ def test_unit_test_custom_stream_wrapper_repeating_chunk(
     print(f"expected_chunk_fail: {expected_chunk_fail}")
 
     if (loop_amount > litellm.REPEATED_STREAMING_CHUNK_LIMIT) and expected_chunk_fail:
+        def _drain():
+            for chunk in response:
+                continue
+
         with pytest.raises(
             (litellm.InternalServerError, litellm.exceptions.MidStreamFallbackError)
         ):
-            for chunk in response:
-                continue
+            _drain()
     else:
         for chunk in response:
             continue

tests/local_testing/test_text_completion.py

@@ -4036,7 +4036,7 @@ def test_async_text_completion_together_ai():
     async def test_get_response():
         try:
             response = await litellm.atext_completion(
-                model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
+                model="together_ai/openai/gpt-oss-20b",
                 prompt="good morning",
                 max_tokens=10,
             )

tests/logging_callback_tests/test_standard_logging_payload.py

@@ -293,7 +293,7 @@ def test_cleanup_timestamps():
     assert all(isinstance(x, float) for x in result)
 
     # Test invalid input
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError, match="start_time is required, got=invalid of type <class 'str'>"):
         StandardLoggingPayloadSetup.cleanup_timestamps(
             "invalid", end_float, completion_float
         )

tests/multi_instance_e2e_tests/test_update_team_e2e.py

@@ -143,7 +143,7 @@ async def test_team_blocking_behavior_multi_instance():
         assert team_info_4001["blocked"] is True, "Team should be blocked after update"
 
         # 8. Make a chat completion request on port 4000 with a new prompt; expect it to be blocked.
-        with pytest.raises(Exception) as excinfo:
+        with pytest.raises(Exception, match="(?i)blocked") as excinfo:
             await chat_completion_on_port(
                 session,
                 key=key,
@@ -157,7 +157,7 @@ async def test_team_blocking_behavior_multi_instance():
         ), f"Expected error indicating team blocked, got: {error_msg}"
 
         # 9. Make a chat completion request on port 4000 with a new prompt; expect it to be blocked.
-        with pytest.raises(Exception) as excinfo:
+        with pytest.raises(Exception, match="(?i)blocked") as excinfo:
             await chat_completion_on_port(
                 session,
                 key=key,
@@ -171,7 +171,7 @@ async def test_team_blocking_behavior_multi_instance():
         ), f"Expected error indicating team blocked, got: {error_msg}"
 
         # 9. Repeat the chat completion request with another new prompt; expect it to be blocked.
-        with pytest.raises(Exception) as excinfo_second:
+        with pytest.raises(Exception, match="(?i)blocked") as excinfo_second:
             await chat_completion_on_port(
                 session,
                 key=key,

tests/ocr_tests/test_ocr_azure_document_intelligence.py

@@ -101,7 +101,7 @@ class TestAzureDocumentIntelligencePagesParam:
             cfg.map_ocr_params({"pages": [True, False]}, {}, "prebuilt-layout")
 
     def test_map_ocr_params_unsupported_type_raises(self, cfg):
-        with pytest.raises(ValueError):
+        with pytest.raises(ValueError, match='based, Mistral-style\\) or a string like'):
             cfg.map_ocr_params({"pages": 5}, {}, "prebuilt-layout")
 
     def test_get_complete_url_appends_pages_query(self, cfg):

tests/openai_endpoints_tests/test_e2e_openai_responses_api.py

@@ -1,5 +1,5 @@
 import httpx
-from openai import OpenAI, BadRequestError
+from openai import OpenAI, BadRequestError, APIStatusError
 import pytest
 
 
@@ -87,7 +87,7 @@ def test_basic_response():
     print("DELETE response=", delete_response)
 
     # expect an error when getting the response again since it was deleted
-    with pytest.raises(Exception):
+    with pytest.raises(APIStatusError):
         get_response = client.responses.retrieve(response.id)
 
 
@@ -195,6 +195,6 @@ def test_cancel_streaming_response():
 
 def test_cancel_invalid_response_id():
     client = get_test_client()
-    with pytest.raises(Exception):
+    with pytest.raises(APIStatusError):
         # Try to cancel a non-existent response ID
         client.responses.cancel("invalid_response_id_12345")

tests/otel_tests/test_e2e_model_access.py

@@ -3,6 +3,7 @@ import asyncio
 import aiohttp
 import json
 from httpx import AsyncClient
+from openai import PermissionDeniedError
 from typing import Any, Optional, List, Literal
 
 
@@ -134,7 +135,7 @@ async def test_model_access_update():
         await mock_chat_completion(session=session, key=key, model="openai/gpt-5.5")
 
         # Should fail with gpt-5-mini
-        with pytest.raises(Exception) as exc_info:
+        with pytest.raises(PermissionDeniedError) as exc_info:
             await mock_chat_completion(
                 session=session, key=key, model="openai/gpt-5-mini"
             )
@@ -157,7 +158,7 @@ async def test_model_access_update():
         )
 
         # Non-OpenAI model should still fail
-        with pytest.raises(Exception) as exc_info:
+        with pytest.raises(PermissionDeniedError) as exc_info:
             await mock_chat_completion(
                 session=session, key=key, model="anthropic/claude-2"
             )
@@ -254,7 +255,7 @@ async def test_team_model_access_update():
         await mock_chat_completion(session=session, key=key, model="openai/gpt-5.5")
 
         # Should fail with gpt-5-mini
-        with pytest.raises(Exception) as exc_info:
+        with pytest.raises(PermissionDeniedError) as exc_info:
             await mock_chat_completion(
                 session=session, key=key, model="openai/gpt-5-mini"
             )
@@ -279,7 +280,7 @@ async def test_team_model_access_update():
         )
 
         # Non-OpenAI model should still fail
-        with pytest.raises(Exception) as exc_info:
+        with pytest.raises(PermissionDeniedError) as exc_info:
             await mock_chat_completion(
                 session=session, key=key, model="anthropic/claude-2"
             )

tests/proxy_admin_ui_tests/test_access_group_team_sync.py

@@ -170,12 +170,15 @@ async def test_a_failed_mirror_takes_the_new_team_row_with_it():
     async with _clean_db() as db:
         await _seed(db, {GROUPS[0]: [], GROUPS[1]: [OTHER_TEAM]})
 
-        with pytest.raises(RuntimeError):
+        async def _blow_up_after_reconcile():
             async with db.tx() as tx:
                 await tx.litellm_teamtable.create(data={"team_id": TEAM, "access_group_ids": [GROUPS[0]]})
                 await reconcile_team_access_group_membership(tx, TEAM)
                 raise RuntimeError("the cache handoff blew up")
 
+        with pytest.raises(RuntimeError):
+            await _blow_up_after_reconcile()
+
         assert await _read(db) == {GROUPS[0]: [], GROUPS[1]: [OTHER_TEAM]}
         assert await db.litellm_teamtable.find_unique(where={"team_id": TEAM}) is None
 

tests/proxy_admin_ui_tests/test_key_management.py

@@ -1340,6 +1340,6 @@ async def test_team_model_alias(prisma_client, requested_model, should_pass):
         }, "Expected model aliases to be present"
     else:
         # Verify the key fails with non-aliased models
-        with pytest.raises(Exception) as exc_info:
+        with pytest.raises(ProxyException) as exc_info:
             await user_api_key_auth(request=request, api_key=f"Bearer {generated_key}")
         assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied

tests/proxy_admin_ui_tests/test_role_based_access.py

@@ -9,7 +9,7 @@ from litellm._uuid import uuid
 from datetime import datetime
 
 from dotenv import load_dotenv
-from fastapi import Request
+from fastapi import HTTPException, Request
 from fastapi.routing import APIRoute
 
 load_dotenv()
@@ -530,7 +530,7 @@ async def test_user_role_permissions(prisma_client, route, user_role, expected_r
             print(f"Auth passed as expected for {route} with role {user_role}")
         else:
             # Should raise an error
-            with pytest.raises(Exception) as exc_info:
+            with pytest.raises((ProxyException, HTTPException)) as exc_info:
                 await user_api_key_auth(request=request, api_key=bearer_token)
             print(f"Auth failed as expected for {route} with role {user_role}")
             print(f"Error message: {str(exc_info.value)}")