Merge main and preserve learned V2 routing behavior

CLAUDE.md

@@ -25,6 +25,8 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
 
 Never test structure of code only function of it
 
+A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
+
 `tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
 
 End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`

litellm-proxy-extras/litellm_proxy_extras/migrations/20260913000000_add_tpd_limit/migration.sql

@@ -0,0 +1,14 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -17,6 +17,7 @@ model LiteLLM_BudgetTable {
   max_parallel_requests Int?
   tpm_limit     BigInt?
   rpm_limit     BigInt?
+  tpd_limit     BigInt?
   model_max_budget Json?
   budget_duration String?
   budget_reset_at DateTime?
@@ -133,6 +134,7 @@ model LiteLLM_TeamTable {
     max_parallel_requests Int?
     tpm_limit     BigInt?
     rpm_limit     BigInt?
+    tpd_limit     BigInt?
     budget_duration String? 
     budget_reset_at DateTime?
     blocked Boolean @default(false)
@@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable {
     max_parallel_requests   Int?
     tpm_limit               BigInt?
     rpm_limit               BigInt?
+    tpd_limit               BigInt?
     budget_duration         String?
     budget_reset_at         DateTime?
     blocked                 Boolean  @default(false)
@@ -438,6 +441,7 @@ model LiteLLM_VerificationToken {
     blocked Boolean?
     tpm_limit     BigInt?
     rpm_limit     BigInt?
+    tpd_limit     BigInt?
     max_budget Float?    
     budget_duration String? 
     budget_reset_at DateTime?
@@ -534,6 +538,7 @@ model LiteLLM_DeletedVerificationToken {
     blocked                 Boolean?
     tpm_limit               BigInt?
     rpm_limit               BigInt?
+    tpd_limit               BigInt?
     max_budget              Float?
     budget_duration         String?
     budget_reset_at         DateTime?

litellm/a2a_protocol/main.py

@@ -21,6 +21,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator
 from litellm.a2a_protocol.utils import A2ARequestUtils
 from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
@@ -507,7 +508,7 @@ async def asend_message(
         prompt_tokens,
         completion_tokens,
         _,
-    ) = A2ARequestUtils.calculate_usage_from_request_response(
+    ) = await asyncify(A2ARequestUtils.calculate_usage_from_request_response)(
         request=request,
         response_dict=response_dict,
     )

litellm/a2a_protocol/streaming_iterator.py

@@ -11,6 +11,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.a2a_protocol.cost_calculator import A2ACostCalculator
 from litellm.a2a_protocol.utils import A2ARequestUtils
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 
 if TYPE_CHECKING:
@@ -99,11 +100,11 @@ class A2AStreamingIterator:
             # Calculate tokens from collected text
             input_message: Final = A2ARequestUtils.get_input_message_from_request(self.request)
             input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
-            prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
+            prompt_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(input_text)
 
             # Use the last (most complete) text from chunks
             output_text: Final = self.collected_text_parts[-1] if self.collected_text_parts else ""
-            completion_tokens: Final = A2ARequestUtils.count_tokens(output_text)
+            completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(output_text)
 
             total_tokens: Final = prompt_tokens + completion_tokens
 

litellm/caching/qdrant_semantic_cache.py

@@ -21,6 +21,7 @@ from litellm.constants import (
     QDRANT_VECTOR_SIZE,
     SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS,
 )
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_str_from_messages,
 )
@@ -255,7 +256,7 @@ class QdrantSemanticCache(BaseCache):
             llm_router = None
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
-        embedding_input: Final = self._embedding_input(prompt, router)
+        embedding_input: Final = await asyncify(self._embedding_input)(prompt, router)
         embedding_call: Final = (
             router.aembedding(
                 model=self.embedding_model,

litellm/caching/redis_semantic_cache.py

@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
 import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_str_from_messages,
 )
@@ -522,7 +523,7 @@ class RedisSemanticCache(BaseCache):
             llm_router = None
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
-        embedding_input: Final = self._embedding_input(prompt, router)
+        embedding_input: Final = await asyncify(self._embedding_input)(prompt, router)
         embedding_call: Final = (
             router.aembedding(
                 model=self.embedding_model,

litellm/constants.py

@@ -1566,6 +1566,8 @@ BASE_MCP_ROUTE: Final = "/mcp"
 
 BATCH_STATUS_POLL_INTERVAL_SECONDS: Final = int(os.getenv("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600))  # 1 hour
 BATCH_STATUS_POLL_MAX_ATTEMPTS: Final = int(os.getenv("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24))  # for 24 hours
+BATCH_TPD_WINDOW_SECONDS: Final = 86400
+BATCH_TPD_DESCRIPTOR_SUFFIX: Final = "_tpd"
 
 HEALTH_CHECK_TIMEOUT_SECONDS: Final = int(os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60))  # 60 seconds
 _background_health_check_max_tokens_env: Final = os.getenv("BACKGROUND_HEALTH_CHECK_MAX_TOKENS")

litellm/cost_calculator.py

@@ -814,6 +814,7 @@ def _select_model_name_for_cost_calc(
             if (
                 entry.get("input_cost_per_token") is not None
                 or entry.get("input_cost_per_second") is not None
+                or entry.get("input_cost_per_query") is not None
                 or entry.get("tiered_pricing") is not None
             ):
                 return_model = router_model_id

litellm/integrations/compression_interception/handler.py

@@ -15,6 +15,7 @@ from typing_extensions import ReadOnly, TypedDict
 from litellm._logging import verbose_logger
 from litellm.compression import compress
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.types.integrations.compression_interception import (
     CompressionInterceptionConfig,
     CompressionSavingsMetadata,
@@ -153,7 +154,7 @@ class CompressionInterceptionLogger(CustomLogger):
 
         self._prune_expired_cache()
 
-        compressed: Final = compress(
+        compressed: Final = await asyncify(compress)(
             messages=messages,
             model=model,
             call_type=CallTypes.anthropic_messages,

litellm/integrations/langsmith.py

@@ -39,6 +39,8 @@ def is_serializable(value):
 
 
 class LangsmithLogger(CustomBatchLogger):
+    preserve_events_added_during_flush = True
+
     def __init__(
         self,
         langsmith_api_key: str | None = None,

litellm/integrations/prometheus.py

@@ -2610,12 +2610,6 @@ class PrometheusLogger(CustomLogger):
             StandardLoggingPayloadSetup,
         )
 
-        if self._should_skip_metrics_for_invalid_key(
-            user_api_key_dict=user_api_key_dict,
-            exception=original_exception,
-        ):
-            return
-
         status_code: Final = self._extract_status_code(exception=original_exception)
 
         try:
@@ -2633,7 +2627,7 @@ class PrometheusLogger(CustomLogger):
                 end_user=user_api_key_dict.end_user_id,
                 user=user_api_key_dict.user_id,
                 user_email=user_api_key_dict.user_email,
-                hashed_api_key=user_api_key_dict.api_key,
+                hashed_api_key=None if status_code == 401 else user_api_key_dict.api_key,
                 api_key_alias=user_api_key_dict.key_alias,
                 team=user_api_key_dict.team_id,
                 team_alias=user_api_key_dict.team_alias,

litellm/litellm_core_utils/get_litellm_params.py

@@ -41,6 +41,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "azure_password",
             "azure_scope",
             "timeout",
+            "client_side_timeout",
             "gcs_bucket_name",
             "bucket_name",
             "vertex_credentials",

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -238,6 +238,8 @@ def get_llm_provider(
             if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
                 raise Exception(f"dynamic_api_key needs to be a string. Got type={type(dynamic_api_key).__name__}")
             return model, custom_llm_provider, dynamic_api_key, api_base
+        if "/" in model and is_registered_custom_provider(provider_prefix):
+            return model.split("/", 1)[1], provider_prefix, dynamic_api_key, api_base
         # check if api base is a known openai compatible endpoint
         if api_base:
             for endpoint in litellm.openai_compatible_endpoints:
@@ -536,6 +538,10 @@ def get_llm_provider(
             )
 
 
+def is_registered_custom_provider(custom_llm_provider: str | None) -> bool:
+    return any(item["provider"] == custom_llm_provider for item in litellm.custom_provider_map)
+
+
 def _dashscope_family_chat_config(custom_llm_provider: str) -> "litellm.DashScopeChatConfig":
     if custom_llm_provider == "qwencloud":
         return litellm.QwenCloudChatConfig()

litellm/litellm_core_utils/litellm_logging.py

@@ -1991,6 +1991,12 @@ class Logging(LiteLLMLoggingBaseClass):
         self.model_call_details["combined_usage_object"] = usage
         self.model_call_details["response_cost"] = response_cost
 
+    def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
+        """Bill a fully streamed response on the failure log when a post-call hook rejects it."""
+        usage: Final = getattr(assembled, "usage", None)
+        if isinstance(usage, Usage):
+            self.record_partial_usage_for_failure(usage, self._response_cost_calculator(result=assembled) or 0.0)
+
     async def dispatch_failure_handlers(
         self,
         exception: Exception,

litellm/litellm_core_utils/streaming_handler.py

@@ -19,6 +19,7 @@ from typing_extensions import NotRequired, TypedDict
 import litellm
 from litellm import verbose_logger
 from litellm._uuid import uuid
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.model_response_utils import (
     is_model_response_stream_empty,
 )
@@ -2247,7 +2248,7 @@ class CustomStreamWrapper:
         if self.sent_last_chunk is True:
             # log the final chunk with accurate streaming values
             try:
-                complete_streaming_response = litellm.stream_chunk_builder(
+                complete_streaming_response = await asyncify(litellm.stream_chunk_builder)(
                     chunks=self.chunks,
                     messages=self.messages,
                     logging_obj=self.logging_obj,

litellm/llms/anthropic/experimental_pass_through/context_management/dispatcher.py

@@ -5,6 +5,7 @@ from collections.abc import Awaitable, Callable
 from typing import TYPE_CHECKING, Final, TypeAlias
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.types.llms.anthropic import AppliedEdit
 
 from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE
@@ -82,9 +83,9 @@ async def apply_context_management(
     """Run edits in order; return a single ``PolyfillResult``.
 
     The dispatcher is async so async editors (``compact_20260112``) can
-    ``await`` the configured summarization model. Sync editors are called
-    inline — ``inspect.iscoroutinefunction`` decides how each editor is
-    invoked.
+    ``await`` the configured summarization model. Sync editors run in a
+    worker thread so their token counts stay off the event loop;
+    ``inspect.iscoroutinefunction`` decides how each editor is invoked.
     """
     edits: Final = _normalize_spec(context_management_spec)
     if not edits:
@@ -121,7 +122,7 @@ async def apply_context_management(
                 user_api_key_auth=user_api_key_auth,
             )
             if editor_is_async
-            else editor(
+            else await asyncify(editor)(
                 model=model,
                 messages=current_messages,
                 tools=tools,

litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py

@@ -20,6 +20,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.types.llms.anthropic import (
     AppliedEdit,
     CompactionBlock,
@@ -1157,7 +1158,7 @@ async def apply_compact_20260112(
 
     # Phase B: threshold check.
     try:
-        current_tokens = _count_effective_tokens(
+        current_tokens = await asyncify(_count_effective_tokens)(
             model=model,
             effective_messages=effective_messages,
             # ``augmented_system`` already carries the prior compaction summary

litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py

@@ -678,7 +678,7 @@ class BaseAnthropicMessagesStreamingIterator:
         """
         from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler
 
-        PassThroughStreamingHandler.schedule_stream_failure_logging(
+        await PassThroughStreamingHandler.schedule_stream_failure_logging(
             litellm_logging_obj=self.litellm_logging_obj,
             endpoint_type=EndpointType.ANTHROPIC,
             request_body=self.request_body,

litellm/llms/fireworks_ai/rerank/transformation.py

@@ -250,8 +250,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig):
 
             rerank_results.append(rerank_result)
 
-        # Use model name as id if no id is provided
-        response_id: Final = raw_response_json.get("id") or raw_response_json.get("model") or str(uuid.uuid4())
+        response_id: Final = raw_response_json.get("id") or str(uuid.uuid4())
 
         return RerankResponse(
             id=response_id,

litellm/llms/vertex_ai/rerank/transformation.py

@@ -4,6 +4,8 @@ Translates from Cohere's `/v1/rerank` input format to Vertex AI Discovery Engine
 Why separate file? Make it easy to see how transformation works
 """
 
+import math
+import uuid
 from collections.abc import Mapping
 from typing import Any, Final
 
@@ -32,6 +34,8 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase):
     Reference: https://cloud.google.com/generative-ai-app-builder/docs/ranking#rank_or_rerank_a_set_of_records_according_to_a_query
     """
 
+    MAX_RECORDS_PER_SEARCH_UNIT = 100
+
     def __init__(self) -> None:
         super().__init__()
 
@@ -208,10 +212,11 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase):
                 RerankResponseResult(index=result["index"], relevance_score=result["relevance_score"])
             )
 
-        # Create meta object
-        meta: Final = RerankResponseMeta(billed_units=RerankBilledUnits(search_units=len(records)))
+        input_record_count: Final = len(request_data.get("records", ()))
+        search_units: Final = math.ceil(input_record_count / self.MAX_RECORDS_PER_SEARCH_UNIT)
+        meta: Final = RerankResponseMeta(billed_units=RerankBilledUnits(search_units=search_units))
 
-        return RerankResponse(id=f"vertex_ai_rerank_{model}", results=rerank_results, meta=meta)
+        return RerankResponse(id=f"vertex_ai_rerank_{uuid.uuid4()}", results=rerank_results, meta=meta)
 
     def get_supported_cohere_rerank_params(self, model: str) -> list:
         return [

litellm/llms/voyage/rerank/transformation.py

@@ -9,6 +9,7 @@ from typing import Any, Final
 
 import httpx
 
+from litellm._uuid import uuid
 from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
 from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
 from litellm.secret_managers.main import get_secret_str
@@ -127,7 +128,7 @@ class VoyageRerankConfig(BaseRerankConfig):
         rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens)
 
         return RerankResponse(
-            id=_json_response.get("id", f"voyage-rerank-{model}"),
+            id=_json_response.get("id") or str(uuid.uuid4()),
             results=transformed_results,
             meta=rerank_meta,
         )

litellm/llms/watsonx/rerank/transformation.py

@@ -191,7 +191,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig):
 
             transformed_results.append(transformed_result)
 
-        response_id: Final = raw_response_json.get("id") or raw_response_json.get("model_id") or str(uuid.uuid4())
+        response_id: Final = raw_response_json.get("id") or str(uuid.uuid4())
 
         # Extract usage information
         _tokens: Final = RerankTokens(

litellm/llms/xai/chat/transformation.py

@@ -219,13 +219,19 @@ class XAIChatConfig(OpenAIGPTConfig):
         litellm_params: dict,
         headers: dict,
     ) -> dict:
-        """
-        Handle https://github.com/BerriAI/litellm/issues/9720
+        """Handle https://github.com/BerriAI/litellm/issues/9720"""
+        if "web_search_options" in optional_params:
+            verbose_logger.warning(
+                "XAI no longer supports web search on /chat/completions (Live Search is deprecated). "
+                "Dropping 'web_search_options'. Use the Responses API for XAI web search."
+            )
 
-        Filter out 'name' from messages
-        """
-        messages = strip_name_from_messages(messages)
-        return super().transform_request(model, messages, optional_params, litellm_params, headers)
+        chat_params: Final = {  # mutable-ok: base transform_request takes a plain dict of optional params
+            key: value for key, value in optional_params.items() if key != "web_search_options"
+        }
+        return super().transform_request(
+            model, strip_name_from_messages(messages), chat_params, litellm_params, headers
+        )
 
     @staticmethod
     def _fix_choice_finish_reason_for_tool_calls(choice: Choices) -> None:

litellm/llms/xai/responses/transformation.py

@@ -3,6 +3,7 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
+from pydantic import TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
@@ -32,6 +33,8 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
+_STR_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+
 
 def _usage_restated_from_xai_ticks(usage: ResponseAPIUsage | None) -> ResponseAPIUsage | None:
     reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None))
@@ -81,30 +84,25 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         - enable_image_understanding
 
         XAI does NOT support search_context_size (OpenAI-specific).
+
+        Domains may come nested under 'filters' (the OpenAI/XAI documented shape) or flat on the tool.
         """
         xai_tool: Final[dict[str, object]] = {"type": "web_search"}
 
-        # Remove search_context_size if present (not supported by XAI)
         if "search_context_size" in tool:
             verbose_logger.info(
                 "XAI does not support 'search_context_size' parameter. Removing it from web_search tool."
             )
 
-        # Handle filters (XAI-specific structure)
-        filters: Final = {}
-        if "allowed_domains" in tool:
-            allowed_domains: Final = tool["allowed_domains"]
-            filters["allowed_domains"] = allowed_domains
+        nested_filters: Final = tool.get("filters")
+        domains: Final = (
+            _STR_MAPPING_ADAPTER.validate_python(nested_filters) if isinstance(nested_filters, Mapping) else tool
+        )
+        filters: Final = {key: domains[key] for key in ("allowed_domains", "excluded_domains") if key in domains}
 
-        if "excluded_domains" in tool:
-            excluded_domains: Final = tool["excluded_domains"]
-            filters["excluded_domains"] = excluded_domains
-
-        # Add filters if any were specified
         if filters:
             xai_tool["filters"] = filters
 
-        # Handle enable_image_understanding (top-level in XAI format)
         if "enable_image_understanding" in tool:
             xai_tool["enable_image_understanding"] = tool["enable_image_understanding"]
 

litellm/main.py

@@ -67,7 +67,7 @@ from litellm.constants import (
 )
 from litellm.exceptions import LiteLLMUnknownProvider
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.litellm_core_utils.asyncify import run_async_function
+from litellm.litellm_core_utils.asyncify import asyncify, run_async_function
 from litellm.litellm_core_utils.audio_utils.utils import (
     calculate_request_duration,
     get_audio_file_for_health_check,
@@ -1072,10 +1072,6 @@ def responses_api_bridge_check(
             mode = "responses"
             model_info["mode"] = mode
 
-        if web_search_options is not None and custom_llm_provider == "xai":
-            model_info["mode"] = "responses"
-            model = model.replace("responses/", "")
-
     except Exception as e:
         verbose_logger.debug("Error getting model info: %s", e)
 
@@ -1084,6 +1080,10 @@ def responses_api_bridge_check(
             mode = "responses"
             model_info["mode"] = mode
 
+    if web_search_options is not None and custom_llm_provider == "xai":
+        model_info["mode"] = "responses"
+        model = model.replace("responses/", "")
+
     # OpenAI/Azure GPT-5 chat-completions that need Responses-only fields (e.g.
     # ``reasoningSummary`` in ``extra_body``) must be bridged; Chat Completions rejects
     # those keys.
@@ -9127,7 +9127,7 @@ async def acount_tokens(
     fallback_messages = messages or []
     if system and fallback_messages:
         fallback_messages = [{"role": "system", "content": system}] + fallback_messages
-    local_count: Final = litellm.token_counter(
+    local_count: Final = await asyncify(litellm.token_counter)(
         model=model,
         messages=fallback_messages,
         tools=tools,

litellm/models/budget.py

@@ -26,6 +26,7 @@ class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase):
     max_parallel_requests: int | None = None
     tpm_limit: int | None = None
     rpm_limit: int | None = None
+    tpd_limit: int | None = None
     model_max_budget: dict | None = None
     budget_duration: str | None = None
     allowed_models: list[str] | None = None  # per-member model scope; empty = inherit team models

litellm/models/team.py

@@ -71,6 +71,7 @@ class TeamBase(LiteLLMPydanticObjectBase):
     metadata: dict | None = None
     tpm_limit: int | None = None
     rpm_limit: int | None = None
+    tpd_limit: int | None = None
     max_budget: float | None = None
     soft_budget: float | None = None
     budget_duration: str | None = None

litellm/models/verification_token.py

@@ -31,6 +31,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase):
     metadata: dict = {}
     tpm_limit: int | None = None
     rpm_limit: int | None = None
+    tpd_limit: int | None = None
     budget_duration: str | None = None
     budget_reset_at: datetime | None = None
     allowed_cache_controls: list | None = []

litellm/proxy/_types.py

@@ -1206,6 +1206,7 @@ class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase):
 
 class KeyRequestBase(GenerateRequestBase):
     key: str | None = None
+    tpd_limit: int | None = None
     default_estimated_output_tokens: PositiveInt | None = None
     default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None
     budget_id: str | None = None
@@ -1891,6 +1892,9 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase):
     )
     tpm_limit: int | None = Field(default=None, description="Max tokens per minute, allowed for this budget id.")
     rpm_limit: int | None = Field(default=None, description="Max requests per minute, allowed for this budget id.")
+    tpd_limit: int | None = Field(
+        default=None, description="Max tokens per day, charged by batch submissions, allowed for this budget id."
+    )
     budget_duration: str | None = Field(
         default=None,
         description="Max duration budget should be set for (e.g. '1hr', '1d', '28d')",
@@ -2067,6 +2071,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
     metadata: dict | None = None
     tpm_limit: int | None = None
     rpm_limit: int | None = None
+    tpd_limit: int | None = None
     max_budget: float | None = None
     soft_budget: float | None = None
     models: list | None = None
@@ -3022,6 +3027,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
     team_alias: str | None = None
     team_tpm_limit: int | None = None
     team_rpm_limit: int | None = None
+    team_tpd_limit: int | None = None
     team_max_budget: float | None = None
     team_soft_budget: float | None = None
     team_models: list = []
@@ -3041,6 +3047,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
     end_user_id: str | None = None
     end_user_tpm_limit: int | None = None
     end_user_rpm_limit: int | None = None
+    end_user_tpd_limit: int | None = None
     end_user_max_budget: float | None = None
     end_user_model_max_budget: dict | None = None
 
@@ -3839,6 +3846,7 @@ class SpendLogsMetadata(TypedDict):
     user_api_key_team_alias: str | None
     spend_logs_metadata: dict | None  # special param to log k,v pairs to spendlogs for a call
     requester_ip_address: str | None
+    user_agent: ReadOnly[str | None]
     litellm_call_id: str | None
     applied_guardrails: list[str] | None
     mcp_tool_call_metadata: StandardLoggingMCPToolCall | None

litellm/proxy/auth/auth_exception_handler.py

@@ -23,6 +23,7 @@ from litellm.proxy.auth.auth_utils import (
     _get_request_ip_address,
     is_invalid_virtual_key_error,
     mark_invalid_virtual_key_error,
+    normalize_request_route,
 )
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
 from litellm.types.services import ServiceTypes
@@ -74,17 +75,28 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
     )
 
 
-def _with_requester_ip_address(request_data: dict[str, object], requester_ip: str | None) -> dict[str, object]:
+def _get_user_agent(request: Request) -> str | None:
+    if "headers" not in request.scope:
+        return None
+    return request.headers.get("user-agent")
+
+
+def _with_client_context(
+    request_data: dict[str, object], requester_ip: str | None, user_agent: str | None
+) -> dict[str, object]:
     """Auth gate rejections are raised before `add_litellm_data_to_request` records the
-    caller IP, so their failure logs would otherwise carry no IP nor key/user identity."""
-    if not requester_ip:
-        return request_data
+    caller IP and User-Agent, so their failure logs would otherwise carry neither."""
     key: Final = "litellm_metadata" if "litellm_metadata" in request_data else "metadata"
     metadata: Final = request_data.get(key)
     base: Final[Mapping[str, object]] = metadata if isinstance(metadata, Mapping) else EMPTY_MAPPING
-    if base.get("requester_ip_address"):
+    stamped: Final = {
+        name: value
+        for name, value in (("requester_ip_address", requester_ip), ("user_agent", user_agent))
+        if value and not base.get(name)
+    }
+    if not stamped:
         return request_data
-    return {**request_data, key: {**base, "requester_ip_address": requester_ip}}  # mutable-ok: logging needs dicts
+    return {**request_data, key: {**base, **stamped}}  # mutable-ok: logging needs dicts
 
 
 class UserAPIKeyAuthExceptionHandler:
@@ -148,6 +160,7 @@ class UserAPIKeyAuthExceptionHandler:
                 request=request,
                 use_x_forwarded_for=general_settings.get("use_x_forwarded_for") is True,
             )
+            user_agent: Final = _get_user_agent(request)
 
             # Log authentication failures before identity seeding and callbacks, so the log
             # survives a raising callback pipeline. Classify and route malformed virtual-key
@@ -172,7 +185,7 @@ class UserAPIKeyAuthExceptionHandler:
             # so the handler is side-effect-free for the caller's identity object.
             user_api_key_dict = resolved_identity.model_copy() if resolved_identity is not None else UserAPIKeyAuth()
             user_api_key_dict.parent_otel_span = parent_otel_span
-            user_api_key_dict.request_route = route
+            user_api_key_dict.request_route = normalize_request_route(route)
             user_api_key_dict.api_key = user_api_key_dict.api_key or UserAPIKeyAuth(api_key=api_key).api_key
 
             # Stamp identity onto the request's server span now, before the request
@@ -200,7 +213,7 @@ class UserAPIKeyAuthExceptionHandler:
 
             # Allow callbacks to transform the error response
             transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
-                request_data=_with_requester_ip_address(request_data, requester_ip),
+                request_data=_with_client_context(request_data, requester_ip, user_agent),
                 original_exception=e,
                 user_api_key_dict=user_api_key_dict,
                 error_type=ProxyErrorTypes.auth_error,

litellm/proxy/auth/team_grants.py

@@ -56,6 +56,7 @@ class TeamGrants(TypedDict, total=False):
     team_alias: ReadOnly[str | None]
     team_tpm_limit: ReadOnly[int | None]
     team_rpm_limit: ReadOnly[int | None]
+    team_tpd_limit: ReadOnly[int | None]
     team_max_budget: ReadOnly[float | None]
     team_soft_budget: ReadOnly[float | None]
     team_spend: ReadOnly[float | None]
@@ -97,6 +98,7 @@ def team_grants(
         team_alias=team_object.team_alias,
         team_tpm_limit=team_object.tpm_limit,
         team_rpm_limit=team_object.rpm_limit,
+        team_tpd_limit=team_object.tpd_limit,
         team_max_budget=team_object.max_budget,
         team_soft_budget=team_object.soft_budget,
         team_spend=team_object.spend,

litellm/proxy/auth/user_api_key_auth.py

@@ -537,6 +537,9 @@ def _apply_budget_limits_to_end_user_params(
     if budget_info.rpm_limit is not None:
         end_user_params["end_user_rpm_limit"] = budget_info.rpm_limit
 
+    if budget_info.tpd_limit is not None:
+        end_user_params["end_user_tpd_limit"] = budget_info.tpd_limit
+
     if budget_info.max_budget is not None:
         end_user_params["end_user_max_budget"] = budget_info.max_budget
 
@@ -621,6 +624,8 @@ def update_valid_token_with_end_user_params(valid_token: UserAPIKeyAuth, end_use
         valid_token.end_user_tpm_limit = end_user_params["end_user_tpm_limit"]
     if end_user_params.get("end_user_rpm_limit") is not None:
         valid_token.end_user_rpm_limit = end_user_params["end_user_rpm_limit"]
+    if end_user_params.get("end_user_tpd_limit") is not None:
+        valid_token.end_user_tpd_limit = end_user_params["end_user_tpd_limit"]
     if end_user_params.get("allowed_model_region") is not None:
         valid_token.allowed_model_region = end_user_params["allowed_model_region"]
     if end_user_params.get("end_user_model_max_budget") is not None:
@@ -2026,6 +2031,7 @@ async def _user_api_key_auth_builder(
             valid_token.end_user_id = end_user_params.get("end_user_id")
             valid_token.end_user_tpm_limit = end_user_params.get("end_user_tpm_limit")
             valid_token.end_user_rpm_limit = end_user_params.get("end_user_rpm_limit")
+            valid_token.end_user_tpd_limit = end_user_params.get("end_user_tpd_limit")
             valid_token.allowed_model_region = end_user_params.get("allowed_model_region")
 
         if valid_token is not None:
@@ -2302,6 +2308,7 @@ async def _user_api_key_auth_builder(
                         spend=valid_token.team_spend,
                         tpm_limit=valid_token.team_tpm_limit,
                         rpm_limit=valid_token.team_rpm_limit,
+                        tpd_limit=valid_token.team_tpd_limit,
                         blocked=valid_token.team_blocked,
                         models=token_team_models,
                         metadata=valid_token.team_metadata,
@@ -2455,6 +2462,7 @@ def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCached
         spend=valid_token.team_spend,
         tpm_limit=valid_token.team_tpm_limit,
         rpm_limit=valid_token.team_rpm_limit,
+        tpd_limit=valid_token.team_tpd_limit,
         blocked=valid_token.team_blocked,
         models=token_team_models,
         metadata=valid_token.team_metadata,

litellm/proxy/common_utils/http_parsing_utils.py

@@ -189,8 +189,9 @@ async def _read_request_body(request: Request | None) -> dict:
 
                     try:
                         parsed_body = json.loads(body_str)
-                    except json.JSONDecodeError:
-                        # If both orjson and json.loads fail, throw a proper error
+                        json.dumps(parsed_body, ensure_ascii=False).encode("utf-8")
+                    except (json.JSONDecodeError, UnicodeEncodeError):
+                        # json.loads accepts lone surrogate escapes that no provider can encode
                         verbose_proxy_logger.error("Invalid JSON payload received: %s", e)
                         raise ProxyException(
                             message=f"Invalid JSON payload: {e}",

litellm/proxy/common_utils/reset_budget_job.py

@@ -7,7 +7,7 @@ from dataclasses import dataclass, field
 from datetime import datetime, timedelta, timezone
 from enum import Enum
 from types import MappingProxyType
-from typing import Final, Literal, Protocol, TypeVar
+from typing import Final, Generic, Literal, Protocol, TypeVar
 
 from typing_extensions import assert_never
 
@@ -68,6 +68,13 @@ from litellm.types.services import ServiceTypes
 
 _RowT = TypeVar("_RowT")
 
+
+@dataclass(frozen=True, slots=True)
+class _RowReset(Generic[_RowT]):
+    row: _RowT
+    spend_decrement: float
+
+
 _LINKED_KEYS_WHERE: Final[Mapping[str, object]] = MappingProxyType({"budget_duration": None, "spend": {"gt": 0}})
 _SPENT_ROWS_WHERE: Final[Mapping[str, object]] = MappingProxyType({"spend": {"gt": 0}})
 
@@ -530,10 +537,9 @@ class ResetBudgetJob:
         )
 
     @staticmethod
-    async def _invalidate_spend_counter(counter_key: str, new_spend: float = 0.0) -> None:
-        """Overwrite a spend counter with the post-reset value (0, or the carried
-        overage when budget rollover is enabled) so a DB-row reset takes effect
-        immediately.
+    async def _invalidate_spend_counter(counter_key: str) -> None:
+        """Drop a spend counter so the next read reseeds from the committed DB
+        row, the only value that includes increments that raced the reset.
 
         Call AFTER the DB write commits. Clearing Redis before the DB
         commit opens a window where get_current_spend reads 0 from Redis
@@ -542,10 +548,10 @@ class ResetBudgetJob:
         try:
             from litellm.proxy.proxy_server import spend_counter_cache
 
-            spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_spend, ttl=60)
+            spend_counter_cache.in_memory_cache.delete_cache(key=counter_key)
             if spend_counter_cache.redis_cache is not None:
                 try:
-                    await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_spend, ttl=60)
+                    await spend_counter_cache.redis_cache.async_delete_cache(key=counter_key)
                 except Exception as redis_err:
                     verbose_proxy_logger.warning(
                         "Failed to reset spend counter %s in Redis: %s. "
@@ -730,8 +736,8 @@ class ResetBudgetJob:
                 uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
 
     async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None:
-        for counter_key, new_spend in cascade.counter_resets:
-            await self._invalidate_spend_counter(counter_key, new_spend=new_spend)
+        for counter_key, _ in cascade.counter_resets:
+            await self._invalidate_spend_counter(counter_key)
         for cache_key in cascade.cache_keys:
             await self._invalidate_user_api_key_cache_entry(cache_key)
 
@@ -842,7 +848,7 @@ class ResetBudgetJob:
         )
         return [LiteLLM_EndUserTable.model_validate(row.model_dump()) for row in rows]
 
-    async def _write_key_reset_updates(self, updated_keys: list[LiteLLM_VerificationToken]) -> None:
+    async def _write_key_reset_updates(self, updated_keys: Sequence[_RowReset[LiteLLM_VerificationToken]]) -> None:
         """
         Write per-row {spend, budget_reset_at} updates for keys.
 
@@ -858,18 +864,18 @@ class ResetBudgetJob:
             reason="reset_budget_write_keys_failure",
         )
 
-    async def _write_key_reset_updates_once(self, updated_keys: list[LiteLLM_VerificationToken]) -> None:
+    async def _write_key_reset_updates_once(self, updated_keys: Sequence[_RowReset[LiteLLM_VerificationToken]]) -> None:
         async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
             for k in updated_keys:
-                if k.token is None:
+                if k.row.token is None:
                     continue
                 uow.keys.queue_spend_reset(
-                    token=k.token,
-                    budget_reset_at=k.budget_reset_at,
-                    spend_decrement=k.max_budget if (k.spend or 0.0) > 0.0 else None,
+                    token=k.row.token,
+                    budget_reset_at=k.row.budget_reset_at,
+                    spend_decrement=k.spend_decrement,
                 )
 
-    async def _write_user_reset_updates(self, updated_users: list[LiteLLM_UserTable]) -> None:
+    async def _write_user_reset_updates(self, updated_users: Sequence[_RowReset[LiteLLM_UserTable]]) -> None:
         """
         Write per-row {spend, budget_reset_at} updates for users.
 
@@ -882,16 +888,16 @@ class ResetBudgetJob:
             reason="reset_budget_write_users_failure",
         )
 
-    async def _write_user_reset_updates_once(self, updated_users: list[LiteLLM_UserTable]) -> None:
+    async def _write_user_reset_updates_once(self, updated_users: Sequence[_RowReset[LiteLLM_UserTable]]) -> None:
         async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
             for u in updated_users:
                 uow.users.queue_spend_reset(
-                    user_id=u.user_id,
-                    budget_reset_at=u.budget_reset_at,
-                    spend_decrement=u.max_budget if (u.spend or 0.0) > 0.0 else None,
+                    user_id=u.row.user_id,
+                    budget_reset_at=u.row.budget_reset_at,
+                    spend_decrement=u.spend_decrement,
                 )
 
-    async def _write_team_reset_updates(self, updated_teams: list[LiteLLM_TeamTable]) -> None:
+    async def _write_team_reset_updates(self, updated_teams: Sequence[_RowReset[LiteLLM_TeamTable]]) -> None:
         """
         Write per-row {spend, budget_reset_at} updates for teams.
 
@@ -904,13 +910,13 @@ class ResetBudgetJob:
             reason="reset_budget_write_teams_failure",
         )
 
-    async def _write_team_reset_updates_once(self, updated_teams: list[LiteLLM_TeamTable]) -> None:
+    async def _write_team_reset_updates_once(self, updated_teams: Sequence[_RowReset[LiteLLM_TeamTable]]) -> None:
         async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
             for t in updated_teams:
                 uow.teams.queue_spend_reset(
-                    team_id=t.team_id,
-                    budget_reset_at=t.budget_reset_at,
-                    spend_decrement=t.max_budget if (t.spend or 0.0) > 0.0 else None,
+                    team_id=t.row.team_id,
+                    budget_reset_at=t.row.budget_reset_at,
+                    spend_decrement=t.spend_decrement,
                 )
 
     def _emit_phase_failure(
@@ -962,18 +968,24 @@ class ResetBudgetJob:
                 reason="reset_budget_read_keys_failure",
             )
             verbose_proxy_logger.debug("Keys to reset %s", _LazyJson(keys_to_reset))
-            updated_keys: Final[list[LiteLLM_VerificationToken]] = []
+            updated_keys: Final[list[_RowReset[LiteLLM_VerificationToken]]] = []
             failed_keys: Final = []
             if keys_to_reset is not None and len(keys_to_reset) > 0:
                 for key in keys_to_reset:
                     try:
+                        pre_reset_spend = float(key.spend or 0.0)
                         updated_key = await ResetBudgetJob._reset_budget_for_key(
                             key=key,
                             current_time=now,
                             reset_settings=self.reset_settings,
                         )
                         if updated_key is not None:
-                            updated_keys.append(updated_key)
+                            updated_keys.append(
+                                _RowReset(
+                                    row=updated_key,
+                                    spend_decrement=pre_reset_spend - float(updated_key.spend or 0.0),
+                                )
+                            )
                         else:
                             failed_keys.append({"key": key, "error": "Returned None without exception"})
                     except Exception as e:
@@ -985,15 +997,15 @@ class ResetBudgetJob:
                 if updated_keys:
                     await self._write_key_reset_updates(updated_keys=updated_keys)
                     for k in updated_keys:
-                        token = getattr(k, "token", None)
+                        token = getattr(k.row, "token", None)
                         if token:
-                            await self._invalidate_spend_counter(f"spend:key:{token}", new_spend=k.spend or 0.0)
+                            await self._invalidate_spend_counter(f"spend:key:{token}")
 
             end_time = time.time()
             outcome: Final = _ChunkOutcome(
                 fetched=len(keys_to_reset) if keys_to_reset else 0,
                 advanced=_count_advanced(
-                    (k.budget_reset_at for k in updated_keys),
+                    (k.row.budget_reset_at for k in updated_keys),
                     cutoff=datetime.now(timezone.utc),
                 ),
             )
@@ -1063,18 +1075,24 @@ class ResetBudgetJob:
                 ),
                 reason="reset_budget_read_users_failure",
             )
-            updated_users: Final[list[LiteLLM_UserTable]] = []
+            updated_users: Final[list[_RowReset[LiteLLM_UserTable]]] = []
             failed_users: Final = []
             if users_to_reset is not None and len(users_to_reset) > 0:
                 for user in users_to_reset:
                     try:
+                        pre_reset_spend = float(user.spend or 0.0)
                         updated_user = await ResetBudgetJob._reset_budget_for_user(
                             user=user,
                             current_time=now,
                             reset_settings=self.reset_settings,
                         )
                         if updated_user is not None:
-                            updated_users.append(updated_user)
+                            updated_users.append(
+                                _RowReset(
+                                    row=updated_user,
+                                    spend_decrement=pre_reset_spend - float(updated_user.spend or 0.0),
+                                )
+                            )
                         else:
                             failed_users.append(
                                 {
@@ -1090,9 +1108,9 @@ class ResetBudgetJob:
                 if updated_users:
                     await self._write_user_reset_updates(updated_users=updated_users)
                     for u in updated_users:
-                        user_id = getattr(u, "user_id", None)
+                        user_id = getattr(u.row, "user_id", None)
                         if user_id:
-                            await self._invalidate_spend_counter(f"spend:user:{user_id}", new_spend=u.spend or 0.0)
+                            await self._invalidate_spend_counter(f"spend:user:{user_id}")
                         if user_id == LITELLM_PROXY_BUDGET_NAME:
                             await self._invalidate_global_proxy_spend_cache()
 
@@ -1100,7 +1118,7 @@ class ResetBudgetJob:
             outcome: Final = _ChunkOutcome(
                 fetched=len(users_to_reset) if users_to_reset else 0,
                 advanced=_count_advanced(
-                    (u.budget_reset_at for u in updated_users),
+                    (u.row.budget_reset_at for u in updated_users),
                     cutoff=datetime.now(timezone.utc),
                 ),
             )
@@ -1172,18 +1190,24 @@ class ResetBudgetJob:
                 ),
                 reason="reset_budget_read_teams_failure",
             )
-            updated_teams: Final[list[LiteLLM_TeamTable]] = []
+            updated_teams: Final[list[_RowReset[LiteLLM_TeamTable]]] = []
             failed_teams: Final = []
             if teams_to_reset is not None and len(teams_to_reset) > 0:
                 for team in teams_to_reset:
                     try:
+                        pre_reset_spend = float(team.spend or 0.0)
                         updated_team = await ResetBudgetJob._reset_budget_for_team(
                             team=team,
                             current_time=now,
                             reset_settings=self.reset_settings,
                         )
                         if updated_team is not None:
-                            updated_teams.append(updated_team)
+                            updated_teams.append(
+                                _RowReset(
+                                    row=updated_team,
+                                    spend_decrement=pre_reset_spend - float(updated_team.spend or 0.0),
+                                )
+                            )
                         else:
                             failed_teams.append(
                                 {
@@ -1199,15 +1223,15 @@ class ResetBudgetJob:
                 if updated_teams:
                     await self._write_team_reset_updates(updated_teams=updated_teams)
                     for t in updated_teams:
-                        team_id = getattr(t, "team_id", None)
+                        team_id = getattr(t.row, "team_id", None)
                         if team_id:
-                            await self._invalidate_spend_counter(f"spend:team:{team_id}", new_spend=t.spend or 0.0)
+                            await self._invalidate_spend_counter(f"spend:team:{team_id}")
 
             end_time = time.time()
             outcome: Final = _ChunkOutcome(
                 fetched=len(teams_to_reset) if teams_to_reset else 0,
                 advanced=_count_advanced(
-                    (t.budget_reset_at for t in updated_teams),
+                    (t.row.budget_reset_at for t in updated_teams),
                     cutoff=datetime.now(timezone.utc),
                 ),
             )

litellm/proxy/db/create_views.py

@@ -80,6 +80,7 @@ async def create_missing_views(db: SupportsRawQueries) -> None:
                 t.max_budget AS team_max_budget,
                 t.tpm_limit AS team_tpm_limit,
                 t.rpm_limit AS team_rpm_limit,
+                t.tpd_limit AS team_tpd_limit,
                 p.project_alias AS project_alias
                 FROM "LiteLLM_VerificationToken" v
                 LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id

litellm/proxy/db/routing_prisma_wrapper.py

@@ -81,6 +81,11 @@ class WriterPinnedClient:
         self.db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) and not db.writer_unavailable else db
 
 
+def writer_wrapper(db: "PrismaWrapper | RoutingPrismaWrapper") -> PrismaWrapper:
+    """Unlike `WriterPinnedClient`, ignores `writer_unavailable`: a raw SQL write has no replica fallback."""
+    return db.writer if isinstance(db, RoutingPrismaWrapper) else db
+
+
 class RoutingPrismaWrapper:
     """
     Routes Prisma operations between a writer and a reader Prisma client.

litellm/proxy/hooks/batch_rate_limiter.py

@@ -18,12 +18,13 @@ Quick summary:
 """
 
 import json
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from datetime import datetime
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, TypeAlias
 
 from fastapi import HTTPException
-from pydantic import BaseModel, Field, TypeAdapter
+from pydantic import BaseModel, Field, TypeAdapter, ValidationError
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -33,6 +34,7 @@ from litellm.batches.batch_utils import (
     _extract_file_access_credentials,
     _iter_batch_input_lines,
 )
+from litellm.constants import BATCH_TPD_DESCRIPTOR_SUFFIX, BATCH_TPD_WINDOW_SECONDS
 from litellm.exceptions import RateLimitErrorCategory
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import (
@@ -55,6 +57,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
 from litellm.proxy.hooks.parallel_request_limiter_v3 import (
     PROJECT_ITPM_DESCRIPTOR_KEY,
     PROJECT_OTPM_DESCRIPTOR_KEY,
+    ReservationAwareIncrementOperation,
     get_or_create_request_stash,
 )
 from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
@@ -92,6 +95,7 @@ else:
 
 
 _BATCH_BODY_ADAPTER: Final = TypeAdapter(dict[str, object])
+_WINDOW_START_ADAPTER: Final[TypeAdapter[int | float | str | None]] = TypeAdapter(int | float | str | None)
 
 IncrementAmounts: TypeAlias = dict[Literal["requests", "tokens"], int]
 
@@ -128,6 +132,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         self,
         internal_usage_cache: InternalUsageCache,
         parallel_request_limiter: ParallelRequestLimiter,
+        time_provider: Callable[[], datetime] | None = None,
     ):
         """
         Initialize the batch rate limiter.
@@ -138,9 +143,11 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         Args:
             internal_usage_cache: Cache for storing rate limit data (auto-injected)
             parallel_request_limiter: Existing rate limiter to integrate with (needs custom injection)
+            time_provider: Clock used for rate limit reset times (defaults to ``datetime.now``)
         """
         self.internal_usage_cache = internal_usage_cache
         self.parallel_request_limiter = parallel_request_limiter
+        self._time_provider: Final = time_provider or datetime.now
         self._warned_unsupported_model_skip = False
 
     def _get_file_bound_batch_model(self, data: dict) -> str | None:
@@ -236,14 +243,48 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         file-bound/top-level routing model this function resolves. Charging
         project quotas here would let a caller bind the file to a model
         without a quota while rows execute against a quota-limited model.
+
+        Scopes with a ``tpd_limit`` (key, team, end user) are charged against a
+        daily token descriptor instead of their per-minute RPM/TPM descriptor,
+        because a batch's rows are scheduled by the provider and never share a
+        minute with the submission. The daily descriptor uses its own key so
+        its 24h window never collides with the online limiter's counters.
         """
-        return self.parallel_request_limiter._create_rate_limit_descriptors(
+        descriptors: Final = self.parallel_request_limiter._create_rate_limit_descriptors(
             user_api_key_dict=user_api_key_dict,
             data=data,
             rpm_limit_type=None,
             tpm_limit_type=None,
             model_has_failures=False,
         )
+        tpd_limits: Final[Mapping[str, tuple[str, int]]] = MappingProxyType(
+            {
+                key: (value, limit)
+                for key, value, limit in (
+                    ("api_key", user_api_key_dict.api_key, user_api_key_dict.tpd_limit),
+                    ("team", user_api_key_dict.team_id, user_api_key_dict.team_tpd_limit),
+                    ("end_user", user_api_key_dict.end_user_id, user_api_key_dict.end_user_tpd_limit),
+                )
+                if value and limit is not None
+            }
+        )
+        if not tpd_limits:
+            return descriptors
+        return [
+            *(d for d in descriptors if d["key"] not in tpd_limits),
+            *(
+                RateLimitDescriptor(
+                    key=f"{key}{BATCH_TPD_DESCRIPTOR_SUFFIX}",
+                    value=value,
+                    rate_limit={
+                        "requests_per_unit": None,
+                        "tokens_per_unit": limit,
+                        "window_size": BATCH_TPD_WINDOW_SECONDS,
+                    },
+                )
+                for key, (value, limit) in tpd_limits.items()
+            ),
+        ]
 
     @staticmethod
     def _project_has_any_io_token_limits(user_api_key_dict: UserAPIKeyAuth) -> bool:
@@ -583,9 +624,14 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         batch_usage: BatchFileUsage,
         limit_type: str,
         requested_model: str | None = None,
+        window_start: int | None = None,
     ) -> NoReturn:
-        """Raise :class:`ProxyRateLimitError` (a 429) for batch rate limit exceeded."""
-        from datetime import datetime
+        """Raise :class:`ProxyRateLimitError` (a 429) for batch rate limit exceeded.
+
+        ``window_start`` is the active counter window's start (unix seconds) when
+        known, so the reset time reflects that window's actual end rather than a
+        full window from now.
+        """
 
         # Find the descriptor for this status. Matching on (key, value) is
         # required, not key alone: a batch can carry several project ITPM/OTPM
@@ -609,9 +655,12 @@ class _PROXY_BatchRateLimiter(CustomLogger):
             descriptors[descriptor_index] if descriptors else {"key": "", "value": "", "rate_limit": None}
         )
 
-        now: Final = datetime.now().timestamp()
-        window_size: Final = self.parallel_request_limiter.window_size
-        reset_time: Final = now + window_size
+        now: Final = self._time_provider().timestamp()
+        window_size: Final = (descriptor.get("rate_limit") or {}).get(
+            "window_size"
+        ) or self.parallel_request_limiter.window_size
+        reset_time: Final = now + window_size if window_start is None else window_start + window_size
+        retry_after: Final = max(0, int(reset_time - now))
         reset_time_formatted: Final = datetime.fromtimestamp(reset_time).strftime("%Y-%m-%d %H:%M:%S UTC")
 
         remaining_display: Final = max(0, status["limit_remaining"])
@@ -643,10 +692,13 @@ class _PROXY_BatchRateLimiter(CustomLogger):
                 if descriptor.get("key") == PROJECT_ITPM_DESCRIPTOR_KEY
                 else batch_usage.total_tokens
             )
+            token_limit_label: Final = (
+                "TPD" if descriptor.get("key", "").endswith(BATCH_TPD_DESCRIPTOR_SUFFIX) else "TPM"
+            )
             detail = (
                 f"Batch rate limit exceeded for {descriptor.get('key', 'unknown')}: {descriptor.get('value', 'unknown')}. "
                 f"Batch contains {batch_token_count} tokens but only {remaining_display} tokens remaining "
-                f"out of {current_limit} TPM limit. "
+                f"out of {current_limit} {token_limit_label} limit. "
                 f"Limit resets at: {reset_time_formatted}"
             )
 
@@ -654,7 +706,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         raise ProxyRateLimitError(
             detail=detail,
             headers={
-                "retry-after": str(window_size),
+                "retry-after": str(retry_after),
                 "rate_limit_type": limit_type,
                 "reset_at": reset_time_formatted,
             },
@@ -712,6 +764,8 @@ class _PROXY_BatchRateLimiter(CustomLogger):
             parent_otel_span=user_api_key_dict.parent_otel_span,
         )
 
+        stash: Final = get_or_create_request_stash()
+        stash.batch_tpd_refund_ops = ()
         if rate_limit_response["overall_code"] == "OVER_LIMIT":
             requested_model: Final = data.get("model") if data else None
             for status in rate_limit_response["statuses"]:
@@ -722,8 +776,70 @@ class _PROXY_BatchRateLimiter(CustomLogger):
                         batch_usage,
                         status["rate_limit_type"],
                         requested_model=requested_model,
+                        window_start=await self._read_tpd_window_start(
+                            status=status, parent_otel_span=user_api_key_dict.parent_otel_span
+                        ),
                     )
 
+        stash.batch_tpd_refund_ops = self._build_tpd_refund_ops(
+            descriptors=descriptors,
+            tokens=batch_usage.total_tokens,
+            reservation_windows=rate_limit_response.get("reservation_windows", frozenset()),
+        )
+
+    async def _read_tpd_window_start(self, status: "RateLimitStatus", parent_otel_span: "Span | None") -> int | None:
+        descriptor_key: Final = status.get("descriptor_key") or ""
+        if not descriptor_key.endswith(BATCH_TPD_DESCRIPTOR_SUFFIX):
+            return None
+        try:
+            window_start: Final = _WINDOW_START_ADAPTER.validate_python(
+                await self.parallel_request_limiter.internal_usage_cache.async_get_cache(
+                    key=f"{{{descriptor_key}:{status.get('descriptor_value') or ''}}}:window",
+                    litellm_parent_otel_span=parent_otel_span,
+                ),
+                strict=True,
+            )
+            return None if window_start is None else int(float(window_start))
+        except (ValidationError, ValueError):
+            return None
+
+    def _build_tpd_refund_ops(
+        self,
+        descriptors: Sequence["RateLimitDescriptor"],
+        tokens: int,
+        reservation_windows: frozenset[tuple[str, str, Literal["redis", "local"]]],
+    ) -> tuple[ReservationAwareIncrementOperation, ...]:
+        """Refund operations for the daily token counters this batch charged.
+
+        The v3 limiter's failure hook applies them when the submission fails
+        after the counters were incremented. Each operation carries the window
+        identity the charge landed in, so the refund is skipped once that
+        window has rolled over.
+        """
+        if tokens <= 0 or not reservation_windows:
+            return ()
+        tpd_descriptors_by_counter: Final[Mapping[str, RateLimitDescriptor]] = MappingProxyType(
+            {
+                self.parallel_request_limiter.create_rate_limit_keys(
+                    descriptor["key"], descriptor["value"], "tokens"
+                ): descriptor
+                for descriptor in descriptors
+                if descriptor["key"].endswith(BATCH_TPD_DESCRIPTOR_SUFFIX)
+            }
+        )
+        return tuple(
+            ReservationAwareIncrementOperation(
+                key=counter_key,
+                increment_value=-tokens,
+                ttl=BATCH_TPD_WINDOW_SECONDS,
+                window_key=f"{{{descriptor['key']}:{descriptor['value']}}}:window",
+                expected_window_start=window_start,
+                reservation_backend=backend,
+            )
+            for counter_key, window_start, backend in sorted(reservation_windows)
+            if (descriptor := tpd_descriptors_by_counter.get(counter_key)) is not None
+        )
+
     async def count_input_file_usage(
         self,
         file_id: str,

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -396,6 +396,8 @@ CacheCounterValue: TypeAlias = int | float | str | bytes
 
 CacheCounterValues: TypeAlias = Sequence[CacheCounterValue | None]
 
+ReservationWindowIdentity: TypeAlias = tuple[str, str, Literal["redis", "local"]]
+
 ParallelGaugeCacheValue: TypeAlias = dict[str, object] | int | float | str | bytes
 
 
@@ -542,6 +544,7 @@ class RequestRateLimiterStash:
         default_factory=frozenset
     )
     batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None
+    batch_tpd_refund_ops: tuple[ReservationAwareIncrementOperation, ...] = ()
     reservation_released: bool = False
 
 
@@ -683,6 +686,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
                 self._batch_rate_limiter = _PROXY_BatchRateLimiter(
                     internal_usage_cache=self.internal_usage_cache,
                     parallel_request_limiter=self,
+                    time_provider=self._time_provider,
                 )
             except Exception as e:
                 verbose_proxy_logger.debug("Could not load batch rate limiter: %s", e)
@@ -1823,6 +1827,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             )
         applied: Final[list[list[AtomicCounterMeta]]] = []
         statuses: Final[list[RateLimitStatus]] = []
+        reservation_windows: Final[set[ReservationWindowIdentity]] = set()  # mutable-ok: filled by the group loop
         raw: list[CacheCounterValue]
 
         for _idx, (keys, args, meta) in enumerate(descriptor_groups):
@@ -1860,11 +1865,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
                 return response
             applied.append(meta)
             statuses.extend(response["statuses"])
+            reservation_windows.update(response.get("reservation_windows", frozenset()))
 
         return RateLimitResponse(
             overall_code="OK",
             statuses=statuses,
-            reservation_windows=frozenset(),
+            reservation_windows=frozenset(reservation_windows),
         )
 
     async def _refund_applied_descriptor_groups(
@@ -4824,6 +4830,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
                 )
                 stash.batch_enqueued_reservation = None
 
+            if stash.batch_tpd_refund_ops:
+                await self.async_increment_reservation_aware_tokens(
+                    pipeline_operations=stash.batch_tpd_refund_ops,
+                    parent_otel_span=user_api_key_dict.parent_otel_span,
+                )
+                stash.batch_tpd_refund_ops = ()
+
             if stash.reservation_released:
                 return
             reserved_tokens: Final = stash.reserved_tokens

litellm/proxy/hooks/proxy_track_cost_callback.py

@@ -652,6 +652,10 @@ async def _update_database_and_spend_counters(
     request_tags: list[str] | None = None,
     model_access_groups: Sequence[str] | None = None,
 ) -> bool:
+    if budget_reservation is not None:
+        await _reconcile_budget_reservation_before_db_update(
+            budget_reservation=budget_reservation, response_cost=response_cost
+        )
     try:
         charged: Final = await proxy_logging_obj.db_spend_update_writer.update_database(
             token=user_api_key,
@@ -709,6 +713,30 @@ async def _update_database_and_spend_counters(
     return True
 
 
+async def _reconcile_budget_reservation_before_db_update(
+    budget_reservation: dict,  # mutable-ok: reconcile_budget_reservation stamps applied_adjustment on the caller's shared reservation dict
+    response_cost: float,
+) -> None:
+    from litellm.proxy.spend_tracking.budget_reservation import reconcile_budget_reservation
+
+    try:
+        await reconcile_budget_reservation(
+            budget_reservation=budget_reservation, actual_cost=response_cost, finalize=False
+        )
+    except Exception:  # noqa: BLE001  # a failed reconcile must not block the spend write; the counters are dropped instead
+        verbose_proxy_logger.warning(
+            "Failed to reconcile budget reservation before persisting spend; invalidating reserved counters"
+        )
+        try:
+            await _invalidate_budget_reservation_counters(budget_reservation=budget_reservation)
+        except Exception:  # noqa: BLE001  # nothing left to try; the finalized stamp below keeps it from being reprocessed
+            verbose_proxy_logger.exception(
+                "Failed to invalidate budget reservation counters after pre-persist reconcile failed"
+            )
+        finally:
+            budget_reservation["finalized"] = True  # rebind-ok: the counter update reads the stamp off the shared dict
+
+
 async def _release_budget_reservation(budget_reservation: dict | None) -> None:
     if budget_reservation is None:
         return

litellm/proxy/management_endpoints/budget_management_endpoints.py

@@ -52,6 +52,7 @@ async def new_budget(
     - max_parallel_requests: Optional[int] - The max number of parallel requests for the budget.
     - tpm_limit: Optional[int] - The tokens per minute limit for the budget.
     - rpm_limit: Optional[int] - The requests per minute limit for the budget.
+    - tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit.
     - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}}
     - budget_reset_at: Optional[datetime] - Datetime when the initial budget is reset. Default is now.
     """
@@ -135,6 +136,7 @@ async def update_budget(
     - max_parallel_requests: Optional[int] - The max number of parallel requests for the budget.
     - tpm_limit: Optional[int] - The tokens per minute limit for the budget.
     - rpm_limit: Optional[int] - The requests per minute limit for the budget.
+    - tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit.
     - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}}
     - budget_reset_at: Optional[datetime] - Update the Datetime when the budget was last reset.
     """
@@ -272,6 +274,7 @@ async def budget_settings(
         "max_parallel_requests": {"type": "Integer"},
         "tpm_limit": {"type": "Integer"},
         "rpm_limit": {"type": "Integer"},
+        "tpd_limit": {"type": "Integer"},
         "budget_duration": {"type": "String"},
         "max_budget": {"type": "Float"},
         "soft_budget": {"type": "Float"},

litellm/proxy/management_endpoints/customer_endpoints.py

@@ -335,6 +335,7 @@ async def new_end_user(
     - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
     - tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute)
     - rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute)
+    - tpd_limit: Optional[int] - Specify tpd limit for a given customer (Tokens per day). Batch submissions are charged against it instead of tpm_limit/rpm_limit
     - model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}}
     - max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer.
     - soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests.

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -1078,7 +1078,9 @@ async def validate_team_id_used_in_service_account_request(
     return True
 
 
-_BUDGET_NUMERIC_KEYS = frozenset(["max_budget", "soft_budget", "max_parallel_requests", "tpm_limit", "rpm_limit"])
+_BUDGET_NUMERIC_KEYS = frozenset(
+    ["max_budget", "soft_budget", "max_parallel_requests", "tpm_limit", "rpm_limit", "tpd_limit"]
+)
 
 
 def _enforce_upperbound_key_params(
@@ -1957,6 +1959,7 @@ async def generate_key_fn(
     - blocked: Optional[bool] - Whether the key is blocked.
     - rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute)
     - tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute)
+    - tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit.
     - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
     - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
     - prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
@@ -2163,6 +2166,7 @@ async def generate_service_account_key_fn(
     - blocked: Optional[bool] - Whether the key is blocked.
     - rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute)
     - tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute)
+    - tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit.
     - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
     - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
     - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
@@ -3192,6 +3196,7 @@ async def update_key_fn(
     - metadata: Optional[dict] - Metadata for key. Example {"team": "core-infra", "app": "app2"}
     - tpm_limit: Optional[int] - Tokens per minute limit
     - rpm_limit: Optional[int] - Requests per minute limit
+    - tpd_limit: Optional[int] - Tokens per day limit, charged by batch submissions instead of tpm_limit/rpm_limit
     - model_rpm_limit: Optional[dict] - Model-specific RPM limits {"gpt-4": 100, "claude-v1": 200}
     - mcp_rpm_limit: Optional[dict] - Per-MCP-server RPM limits, keyed by MCP server name {"github": 100, "slack": 200}
     - tag_rpm_limit: Optional[dict] - Per-request-tag RPM limits, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; absent tags fall back to the key-level rpm limit.
@@ -4355,6 +4360,7 @@ async def generate_key_helper_fn(
     metadata: dict | None = {},
     tpm_limit: int | None = None,
     rpm_limit: int | None = None,
+    tpd_limit: int | None = None,
     query_type: Literal["insert_data", "update_data"] = "insert_data",
     update_key_values: dict | None = None,
     key_alias: str | None = None,
@@ -4503,6 +4509,7 @@ async def generate_key_helper_fn(
             "metadata": metadata_json,
             "tpm_limit": tpm_limit,
             "rpm_limit": rpm_limit,
+            "tpd_limit": tpd_limit,
             "budget_duration": key_budget_duration,
             "budget_reset_at": key_reset_at,
             "allowed_cache_controls": allowed_cache_controls,

litellm/proxy/management_endpoints/management_v1/budgets.py

@@ -58,6 +58,7 @@ class BudgetListItem(BaseModel):
     soft_budget: float | None = None
     tpm_limit: int | None = None
     rpm_limit: int | None = None
+    tpd_limit: int | None = None
     budget_duration: str | None = None
     budget_reset_at: datetime | None = None
     created_at: datetime
@@ -123,7 +124,7 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
 
 BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec(
     resource="budgets",
-    sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at")),
+    sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")),
     searchable=frozenset(("budget_id",)),
     filters=BUDGET_FILTERS,
     default_sort=(SortKey(field="created_at", descending=True),),
@@ -154,7 +155,7 @@ async def list_budgets(
     way to page, sort or filter it.
 
     `sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`,
-    `rpm_limit` or `created_at`, each optionally prefixed with `-` for descending,
+    `rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending,
     and defaults to `-created_at`. `budget_id` is appended to every sort as the
     tiebreaker. `q` is a case-insensitive substring match on `budget_id`.
     `page_size` defaults to 50 and is capped at 100. Filters are

litellm/proxy/management_endpoints/organization_endpoints.py

@@ -362,6 +362,7 @@ async def new_organization(
     - max_budget: *Optional[float]* - Max budget for org
     - tpm_limit: *Optional[int]* - Max tpm limit for org
     - rpm_limit: *Optional[int]* - Max rpm limit for org
+    - tpd_limit: *Optional[int]* - Max tokens per day stored on the org budget. Batch submissions enforce tpd_limit at the key, team and end user scopes only.
     - model_rpm_limit: *Optional[Dict[str, int]]* - The RPM (Requests Per Minute) limit per model for this organization.
     - model_tpm_limit: *Optional[Dict[str, int]]* - The TPM (Tokens Per Minute) limit per model for this organization.
     - max_parallel_requests: *Optional[int]* - [Not Implemented Yet] Max parallel requests for org

litellm/proxy/management_endpoints/team_endpoints.py

@@ -1217,6 +1217,7 @@ async def new_team(
     - mcp_rpm_limit: Optional[Dict[str, int]] - Per-MCP-server RPM limit for this team, keyed by MCP server name (alias if set, else the configured name). Example: {"github": 100, "slack": 200}. Applied across all keys for this team.
     - tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit
     - rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit
+    - tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit
     - rpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of RPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating RPM, or "best_effort_throughput" for best effort enforcement.
     - tpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of TPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating TPM, or "best_effort_throughput" for best effort enforcement.
     - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget
@@ -1969,6 +1970,7 @@ async def update_team(
     - metadata: Optional[dict] - Metadata for team, store information for team. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
     - tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit
     - rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit
+    - tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit
     - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget
     - soft_budget: Optional[float] - The soft budget threshold for the team. If max_budget is set (either in the request or existing), soft_budget must be strictly lower than max_budget. Can be set independently if max_budget is not set.
     - budget_duration: Optional[str] - The duration of the budget for the team. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets)

litellm/proxy/management_helpers/access_group_key_sync.py

@@ -38,7 +38,7 @@ from litellm.proxy._types import (
 from litellm.proxy.auth.auth_checks import (
     _delete_cache_access_object,  # pyright: ignore[reportPrivateUsage]  # the access-group endpoints reach for this same cache primitive
 )
-from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
+from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
 from litellm.repositories.table_repositories import AccessGroupRepository
 
 
@@ -75,7 +75,7 @@ _REPOINT_KEY_SQL: Final = (
 def _raw_executor(prisma_client: object) -> _RawExecutor:
     """Narrow the untyped Prisma client down to the raw-query call this module makes, pinned to the writer."""
     db: Final = AccessGroupRepository(prisma_client).prisma_client.db  # pyright: ignore[reportAny]  # untyped Prisma client
-    return WriterPinnedClient(db).db  # pyright: ignore[reportAny, reportReturnType]  # untyped Prisma client behind the pin
+    return writer_wrapper(db)  # pyright: ignore[reportAny, reportReturnType]  # untyped Prisma client behind the pin
 
 
 async def _invalidate_access_group_cache(access_group_id: str) -> None:

litellm/proxy/management_helpers/access_group_model_sync.py

@@ -10,7 +10,7 @@ from typing import Final, Protocol
 
 from pydantic import BaseModel
 
-from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
+from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
 from litellm.proxy.management_helpers.access_group_team_sync import invalidate_access_group_caches
 from litellm.repositories.table_repositories import AccessGroupRepository
 from litellm.router import Router
@@ -56,7 +56,7 @@ _REMOVE_MODEL_NAME_SQL: Final = (
 
 def _raw_executor(prisma_client: object) -> _RawExecutor:
     db: Final = AccessGroupRepository(prisma_client).prisma_client.db  # pyright: ignore[reportAny]  # untyped Prisma client
-    return WriterPinnedClient(db).db  # pyright: ignore[reportAny, reportReturnType]  # untyped Prisma client behind the pin
+    return writer_wrapper(db)  # pyright: ignore[reportAny, reportReturnType]  # untyped Prisma client behind the pin
 
 
 def _config_sourced_sibling(llm_router: Router, deployment_id: str, model_id: str) -> bool:

litellm/proxy/pass_through_endpoints/streaming_handler.py

@@ -8,6 +8,7 @@ import httpx
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.proxy._types import PassThroughEndpointLoggingResultValues
@@ -60,7 +61,7 @@ class PassThroughStreamingHandler:
             litellm_logging_obj._update_completion_start_time(completion_start_time=datetime.now())
 
     @staticmethod
-    def schedule_stream_failure_logging(
+    async def schedule_stream_failure_logging(
         litellm_logging_obj: LiteLLMLoggingObj,
         endpoint_type: EndpointType,
         request_body: dict[str, object],
@@ -68,7 +69,7 @@ class PassThroughStreamingHandler:
         exception: Exception,
         stream_context: PassThroughStreamContext | None = None,
     ) -> None:
-        PassThroughStreamingHandler._record_partial_usage_for_failure(
+        await asyncify(PassThroughStreamingHandler._record_partial_usage_for_failure)(
             litellm_logging_obj=litellm_logging_obj,
             endpoint_type=endpoint_type,
             request_body=request_body,
@@ -222,7 +223,7 @@ class PassThroughStreamingHandler:
             verbose_proxy_logger.error("Error in chunk_processor: %s", e)
             if response.status_code < 400:
                 logging_scheduled = True
-                PassThroughStreamingHandler.schedule_stream_failure_logging(
+                await PassThroughStreamingHandler.schedule_stream_failure_logging(
                     litellm_logging_obj=litellm_logging_obj,
                     endpoint_type=endpoint_type,
                     request_body=resolved_request_body,
@@ -292,7 +293,7 @@ class PassThroughStreamingHandler:
             (
                 standard_logging_response_object,
                 kwargs,
-            ) = PassThroughStreamingHandler._build_passthrough_logging_result(
+            ) = await asyncify(PassThroughStreamingHandler._build_passthrough_logging_result)(
                 litellm_logging_obj=litellm_logging_obj,
                 passthrough_success_handler_obj=passthrough_success_handler_obj,
                 url_route=url_route,
@@ -334,8 +335,8 @@ class PassThroughStreamingHandler:
         Synchronous, CPU-bound reconstruction of the standard logging payload
         from collected raw SSE bytes. Extracted from
         _route_streaming_logging_to_handler so the per-endpoint dispatch can
-        be unit-tested in isolation. Still invoked synchronously on the event
-        loop; an off-loop dispatch is a future change, not part of this PR.
+        be unit-tested in isolation. The async callers run it in a worker
+        thread so the token counts inside stay off the event loop.
         """
         all_chunks: Final = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes)
         standard_logging_response_object: PassThroughEndpointLoggingResultValues | None = None

litellm/proxy/proxy_server.py

@@ -3061,7 +3061,7 @@ async def _reconcile_budget_reservation_for_counter_update(
     budget_reservation: dict | None,
     response_cost: float | None,
 ) -> set[str]:
-    if budget_reservation is None:
+    if budget_reservation is None or budget_reservation.get("finalized") is True:
         return set()
 
     from litellm.proxy.spend_tracking.budget_reservation import (

litellm/proxy/schema.prisma

@@ -17,6 +17,7 @@ model LiteLLM_BudgetTable {
   max_parallel_requests Int?
   tpm_limit     BigInt?
   rpm_limit     BigInt?
+  tpd_limit     BigInt?
   model_max_budget Json?
   budget_duration String?
   budget_reset_at DateTime?
@@ -133,6 +134,7 @@ model LiteLLM_TeamTable {
     max_parallel_requests Int?
     tpm_limit     BigInt?
     rpm_limit     BigInt?
+    tpd_limit     BigInt?
     budget_duration String? 
     budget_reset_at DateTime?
     blocked Boolean @default(false)
@@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable {
     max_parallel_requests   Int?
     tpm_limit               BigInt?
     rpm_limit               BigInt?
+    tpd_limit               BigInt?
     budget_duration         String?
     budget_reset_at         DateTime?
     blocked                 Boolean  @default(false)
@@ -438,6 +441,7 @@ model LiteLLM_VerificationToken {
     blocked Boolean?
     tpm_limit     BigInt?
     rpm_limit     BigInt?
+    tpd_limit     BigInt?
     max_budget Float?    
     budget_duration String? 
     budget_reset_at DateTime?
@@ -534,6 +538,7 @@ model LiteLLM_DeletedVerificationToken {
     blocked                 Boolean?
     tpm_limit               BigInt?
     rpm_limit               BigInt?
+    tpd_limit               BigInt?
     max_budget              Float?
     budget_duration         String?
     budget_reset_at         DateTime?

litellm/proxy/spend_tracking/budget_reservation.py

@@ -959,8 +959,9 @@ async def _set_reserved_entries_actual_cost(
 
 async def _reseed_reserved_entry(item: _EntryAdjustment, actual_cost: float) -> None:
     """Post-call reconcile / release of a counter that was flushed, expired or reseeded between reservation and
-    reconcile: the optimistic delta no longer applies, so reseed from the DB floor (which cannot include this
-    request's cost yet) and add the settled cost, since increment_spend_counters skips reserved keys."""
+    reconcile: the optimistic delta no longer applies, so reseed from the DB floor and add the settled cost, since
+    increment_spend_counters skips reserved keys. The reconcile runs before this request's spend is enqueued to the
+    DB, so the reseeded floor excludes it."""
     from litellm.proxy.proxy_server import _increment_spend_counter_cache, reseed_spend_counter_from_db
 
     reseeded: Final = await reseed_spend_counter_from_db(counter_key=item.counter_key)

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -157,6 +157,7 @@ def _get_spend_logs_metadata(
             user_api_key_team_alias=None,
             spend_logs_metadata=None,
             requester_ip_address=None,
+            user_agent=None,
             additional_usage_values=None,
             applied_guardrails=None,
             status="success",

litellm/proxy/utils.py

@@ -85,7 +85,11 @@ from litellm._logging import _redact_string, verbose_proxy_logger
 from litellm._service_logger import ServiceLogging, ServiceTypes
 from litellm.caching.caching import DualCache, RedisCache
 from litellm.caching.dual_cache import LimitedSizeOrderedDict
-from litellm.exceptions import RejectedRequestError, SensitiveDataRouteException
+from litellm.exceptions import (
+    GuardrailRaisedException,
+    RejectedRequestError,
+    SensitiveDataRouteException,
+)
 from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     ModifyResponseException,
@@ -901,6 +905,9 @@ def _call_type_for_route(route: str | None) -> str | None:
     return call_types[0].value if len(operations) == 1 else None
 
 
+_PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException)
+
+
 def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, object]:
     """Failure-path callbacks run after ``litellm_logging_obj`` is popped from
     request_data (it is not serialisable), so the caller merges these fields
@@ -2991,6 +2998,7 @@ class ProxyLogging:
             - Authentication Errors from user_api_key_auth
             - HTTP HTTPException (rate limit errors)
             - ProxyException (guardrail blocks, budget / rate-limit errors)
+            - GuardrailRaisedException (guardrail blocks / guardrail failures)
         """
 
         #########################################################
@@ -3005,9 +3013,7 @@ class ProxyLogging:
         if not (RouteChecks.is_llm_api_route(route) or RouteChecks.is_info_route(route)):
             return False
 
-        return isinstance(original_exception, (HTTPException, ProxyException)) or (
-            error_type == ProxyErrorTypes.auth_error
-        )
+        return isinstance(original_exception, _PROXY_ONLY_LLM_API_ERRORS) or (error_type == ProxyErrorTypes.auth_error)
 
     async def _handle_logging_proxy_only_error(
         self,
@@ -3563,8 +3569,9 @@ class ProxyLogging:
                     yield chunk
             except (GeneratorExit, asyncio.CancelledError):
                 raise
-            except Exception:
-                ProxyLogging._fire_deferred_stream_logging(request_data)
+            except Exception as e:
+                if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e):
+                    ProxyLogging._fire_deferred_stream_logging(request_data)
                 raise
             ProxyLogging._fire_deferred_stream_logging(request_data)
             return
@@ -3638,8 +3645,9 @@ class ProxyLogging:
                 yield chunk
         except (GeneratorExit, asyncio.CancelledError):
             raise
-        except Exception:
-            ProxyLogging._fire_deferred_stream_logging(request_data)
+        except Exception as e:
+            if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e):
+                ProxyLogging._fire_deferred_stream_logging(request_data)
             raise
 
         # Fire deferred logging AFTER all guardrail end-of-stream blocks
@@ -3735,6 +3743,23 @@ class ProxyLogging:
             logging_obj._deferred_stream_complete_args = None
             asyncio.create_task(_deferred_cb(*_args))
 
+    @staticmethod
+    def _discard_deferred_stream_logging_for_failure(request_data: Mapping[str, object], error: Exception) -> bool:
+        """Drop the parked success dispatch for an assembled chat stream that ends in an error
+        ``post_call_failure_hook`` logs as a failure, billing its usage on the failure row instead.
+        Returns False when the parked dispatch should still be flushed by the caller."""
+        logging_obj: Final = request_data.get("litellm_logging_obj")
+        if not isinstance(logging_obj, Logging):
+            return False
+        _args: Final[tuple[object, ...] | None] = getattr(logging_obj, "_deferred_stream_complete_args", None)
+        assembled: Final = _args[0] if _args else None
+        if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS) or not isinstance(assembled, ModelResponse):
+            return False
+        logging_obj._on_deferred_stream_complete = None
+        logging_obj._deferred_stream_complete_args = None
+        logging_obj.record_assembled_response_for_failure(assembled)
+        return True
+
     async def _arelease_max_parallel_requests_on_disconnect(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -4295,7 +4320,8 @@ class PrismaClient:
                             t.spend AS team_spend,
                             t.max_budget AS team_max_budget,
                             t.tpm_limit AS team_tpm_limit,
-                            t.rpm_limit AS team_rpm_limit
+                            t.rpm_limit AS team_rpm_limit,
+                            t.tpd_limit AS team_tpd_limit
                             FROM "LiteLLM_VerificationToken" v
                             LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id;
                         """,
@@ -4734,6 +4760,7 @@ class PrismaClient:
                             t.soft_budget AS team_soft_budget,
                             t.tpm_limit AS team_tpm_limit,
                             t.rpm_limit AS team_rpm_limit,
+                            t.tpd_limit AS team_tpd_limit,
                             t.models AS team_models,
                             t.metadata AS team_metadata,
                             t.blocked AS team_blocked,
@@ -4751,6 +4778,7 @@ class PrismaClient:
                             b.max_budget AS litellm_budget_table_max_budget,
                             b.tpm_limit AS litellm_budget_table_tpm_limit,
                             b.rpm_limit AS litellm_budget_table_rpm_limit,
+                            b.tpd_limit AS litellm_budget_table_tpd_limit,
                             b.model_max_budget as litellm_budget_table_model_max_budget,
                             b.soft_budget as litellm_budget_table_soft_budget,
                             o.metadata as organization_metadata,

litellm/repositories/unit_of_work.py

@@ -24,12 +24,8 @@ from typing import Final
 from litellm.repositories.prisma_protocols import BatchTable, PrismaBatch
 
 
-def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float | None) -> Mapping[str, object]:
-    spend: Final[object] = (
-        {"decrement": spend_decrement}  # mutable-ok: prisma update payload must be a dict
-        if spend_decrement is not None
-        else 0
-    )
+def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float) -> Mapping[str, object]:
+    spend: Final[object] = {"decrement": spend_decrement}  # mutable-ok: prisma update payload must be a dict
     return {"spend": spend, "budget_reset_at": budget_reset_at}  # mutable-ok: prisma update payload must be a dict
 
 
@@ -37,9 +33,7 @@ def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float |
 class KeySpendResetWrites:
     table: BatchTable
 
-    def queue_spend_reset(
-        self, token: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
-    ) -> None:
+    def queue_spend_reset(self, token: str, budget_reset_at: datetime | None, spend_decrement: float) -> None:
         self.table.update(
             where={"token": token},  # mutable-ok: prisma where filter must be a dict
             data=_spend_reset_data(budget_reset_at, spend_decrement),
@@ -50,9 +44,7 @@ class KeySpendResetWrites:
 class UserSpendResetWrites:
     table: BatchTable
 
-    def queue_spend_reset(
-        self, user_id: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
-    ) -> None:
+    def queue_spend_reset(self, user_id: str, budget_reset_at: datetime | None, spend_decrement: float) -> None:
         self.table.update(
             where={"user_id": user_id},  # mutable-ok: prisma where filter must be a dict
             data=_spend_reset_data(budget_reset_at, spend_decrement),
@@ -63,9 +55,7 @@ class UserSpendResetWrites:
 class TeamSpendResetWrites:
     table: BatchTable
 
-    def queue_spend_reset(
-        self, team_id: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
-    ) -> None:
+    def queue_spend_reset(self, team_id: str, budget_reset_at: datetime | None, spend_decrement: float) -> None:
         self.table.update(
             where={"team_id": team_id},  # mutable-ok: prisma where filter must be a dict
             data=_spend_reset_data(budget_reset_at, spend_decrement),

litellm/router.py

@@ -79,7 +79,10 @@ from litellm.litellm_core_utils.core_helpers import (
 from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.litellm_core_utils.dd_tracing import tracer
-from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
+from litellm.litellm_core_utils.get_llm_provider_logic import (
+    declared_authenticating_provider,
+    is_registered_custom_provider,
+)
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.ptu_pricing import (
     PTU_COST_ATTRIBUTION_ENV_VAR,
@@ -167,6 +170,7 @@ from litellm.router_utils.cooldown_handlers import (
     _get_cooldown_deployments,
     _set_cooldown_deployments,
     is_advisor_orchestration_failure,
+    is_caller_timeout_408,
 )
 from litellm.router_utils.fallback_event_handlers import (
     AttemptedFallbackTargets,
@@ -3773,7 +3777,16 @@ class Router:
         self, deployment: dict, kwargs: dict, function_name: str | None = None
     ) -> Deployment:
         """
-        Handle clientside credential
+        Build a per-request Deployment carrying the caller-supplied api_key/api_base,
+        with its own stable id for cooldown, logging, and cost-map identity.
+
+        This deployment is deliberately never registered with the router (no
+        upsert_deployment/add_deployment call): doing so used to add it to
+        self.model_list under the shared model_name, which made a request-scoped,
+        caller-supplied provider credential a permanent, load-balanced deployment
+        that every other caller of that model group could be routed onto. Its
+        pricing is still registered directly, so a custom price configured on the
+        underlying deployment still applies to this call.
         """
         model_info: Final = deployment.get("model_info", {}).copy()
         litellm_params: Final = deployment["litellm_params"].copy()
@@ -3792,7 +3805,7 @@ class Router:
             litellm_params=LiteLLM_Params(**dynamic_litellm_params),
             model_info=model_info,
         )
-        self.upsert_deployment(deployment=deployment_pydantic_obj)  # add new deployment to router
+        Router._register_deployment_pricing(deployment=deployment_pydantic_obj)
         return deployment_pydantic_obj
 
     @staticmethod
@@ -8297,6 +8310,13 @@ class Router:
             litellm_params: Final = kwargs.get("litellm_params", {})
             _model_info: Final = litellm_params.get("model_info", {})
 
+            if is_caller_timeout_408(kwargs, exception_status):
+                verbose_router_logger.debug(
+                    "Router: Exiting 'deployment_callback_on_failure' without cooldown. "
+                    "A timeout the caller set caused this 408, not the deployment's health."
+                )
+                return False
+
             exception_headers: Final = litellm.litellm_core_utils.exception_mapping_utils._get_response_headers(
                 original_exception=exception
             )
@@ -9546,8 +9566,10 @@ class Router:
             )
             # done reading model["litellm_params"]
             # Check if provider is supported: either in enum or JSON-configured
-            if custom_llm_provider not in litellm.provider_list and not JSONProviderRegistry.exists(
-                custom_llm_provider
+            if (
+                custom_llm_provider not in litellm.provider_list
+                and not JSONProviderRegistry.exists(custom_llm_provider)
+                and not is_registered_custom_provider(custom_llm_provider)
             ):
                 raise Exception(f"Unsupported provider - {custom_llm_provider}")
 
@@ -9693,40 +9715,7 @@ class Router:
         # initialize client
         self._add_deployment(deployment=deployment)
 
-        _model_info_dict: Final[dict] = deployment.model_info.model_dump(exclude_none=True)
-        for field in CustomPricingLiteLLMParams.model_fields:
-            field_value = deployment.litellm_params.get(field)
-            if field_value is not None:
-                _model_info_dict[field] = field_value
-
-        Router._inherit_builtin_base_rates_for_off_peak(
-            model_info=_model_info_dict,
-            backend_model=deployment.litellm_params.model,
-            custom_llm_provider=deployment.litellm_params.custom_llm_provider,
-        )
-        if _model_info_dict.get("input_cost_per_token") is not None:
-            Router._inherit_builtin_cache_pricing(
-                model_info=_model_info_dict,
-                backend_model=deployment.litellm_params.model,
-                custom_llm_provider=deployment.litellm_params.custom_llm_provider,
-            )
-        Router._inherit_builtin_tiered_output_rate(
-            model_info=_model_info_dict,
-            backend_model=deployment.litellm_params.model,
-            custom_llm_provider=deployment.litellm_params.custom_llm_provider,
-        )
-
-        # Register custom pricing in litellm.model_cost.
-        # Mirrors _create_deployment() logic to ensure dynamically-added deployments
-        # (e.g., loaded from DB) also have their custom pricing registered.
-        # Without this, _is_model_cost_zero() cannot detect explicitly-configured
-        # zero-cost models, causing budget checks to block free models.
-        Router._register_deployment_in_model_cost(
-            model_id=deployment.model_info.id,
-            model_info=_model_info_dict,
-            model=deployment.litellm_params.model,
-            custom_llm_provider=deployment.litellm_params.custom_llm_provider,
-        )
+        Router._register_deployment_pricing(deployment=deployment)
 
         # add to model names
         self._add_model_to_list_and_index_map(model=_deployment, model_id=deployment.model_info.id)
@@ -9988,6 +9977,21 @@ class Router:
         )
         return model_info
 
+    @staticmethod
+    def _register_deployment_pricing(deployment: Deployment) -> None:
+        """Register a deployment's custom/inherited pricing in ``litellm.model_cost``.
+
+        Takes only a ``Deployment``, so it registers pricing for a deployment that
+        is never added to ``self.model_list`` (a per-request client-side-credential
+        deployment) just as readily as one that is.
+        """
+        Router._register_deployment_in_model_cost(
+            model_id=deployment.model_info.id,
+            model_info=Router._deployment_model_cost_payload(deployment),
+            model=deployment.litellm_params.model,
+            custom_llm_provider=deployment.litellm_params.custom_llm_provider,
+        )
+
     @staticmethod
     def _register_deployment_in_model_cost(
         *,

litellm/router_strategy/complexity_router/capability_classifier.py

@@ -202,10 +202,15 @@ def capability_classifier_system_prompt(mode: Literal["json_schema", "json_objec
     )
 
 
-def parse_capability_classifier_verdict(content: str) -> CapabilityClassifierVerdict:
-    """Parse raw JSON or the fenced JSON shape tolerated by Switchyard."""
+def unwrap_classifier_json(content: str) -> str:
+    """Remove the optional Markdown fence without repairing or weakening verdict JSON."""
     text: Final = content.strip()
     if not text.startswith("```"):
-        return CapabilityClassifierVerdict.model_validate_json(text)
+        return text
     unfenced: Final = text.removeprefix("```").removeprefix("json").lstrip("\n\r")
-    return CapabilityClassifierVerdict.model_validate_json(unfenced.removesuffix("```").strip())
+    return unfenced.removesuffix("```").strip()
+
+
+def parse_capability_classifier_verdict(content: str) -> CapabilityClassifierVerdict:
+    """Parse raw JSON or the fenced JSON shape tolerated by Switchyard."""
+    return CapabilityClassifierVerdict.model_validate_json(unwrap_classifier_json(content))

litellm/router_strategy/complexity_router/complexity_router.py

@@ -81,6 +81,7 @@ from .capability_classifier import (
     capability_classifier_response_format,
     capability_classifier_system_prompt,
     parse_capability_classifier_verdict,
+    unwrap_classifier_json,
 )
 from .classification_rubrics import BUSINESS_TIER_CRITERIA, calibration_examples_section
 from .config import (
@@ -103,7 +104,7 @@ from .config import (
     CustomDimension,
     TierDefinition,
 )
-from .llm_v2 import LLMV2TaskContext, LLMV2Verdict, llm_v2_response_format
+from .llm_v2 import LLM_V2_PROMPT_VERSION, LLMV2Decision, LLMV2TaskContext, LLMV2Verdict, llm_v2_response_format
 from .stall_detector import detect_stalled_task
 
 if TYPE_CHECKING:
@@ -1017,16 +1018,41 @@ class ClassificationOutcome(NamedTuple):
     ]
     classifier_cost: float | None = None
     capability_forecast: CapabilityClassifierForecast | None = None
+    llm_v2_forecast: LLMV2Decision | None = None
 
 
 def _with_signal(outcome: ClassificationOutcome, signal: str | None) -> ClassificationOutcome:
     return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal))
 
 
-def _with_capability_forecast(
+def _with_llm_v2_forecast(
+    decision: StandardLoggingRoutingDecision, forecast: LLMV2Decision
+) -> StandardLoggingRoutingDecision:
+    """Preserve full numeric precision for both solver forecasts and the applied policy."""
+    enriched: Final[StandardLoggingRoutingDecision] = {
+        **decision,
+        "classifier_efficient_p_solve": forecast.verdict.forecasts.efficient.p_solve,
+        "classifier_capable_p_solve": forecast.verdict.forecasts.capable.p_solve,
+        **({"classifier_max_quality_gap": forecast.max_quality_gap} if forecast.selective_decision is None else {}),
+        "classifier_prompt_version": LLM_V2_PROMPT_VERSION,
+    }
+    if forecast.calibration_version is None:
+        return enriched
+    calibrated: Final[StandardLoggingRoutingDecision] = {
+        **enriched,
+        "classifier_calibrated_efficient_p_solve": forecast.efficient,
+        "classifier_calibrated_capable_p_solve": forecast.capable,
+        "classifier_calibration_version": forecast.calibration_version,
+    }
+    return calibrated
+
+
+def _with_classifier_forecast(
     decision: StandardLoggingRoutingDecision, outcome: ClassificationOutcome
 ) -> StandardLoggingRoutingDecision:
-    """Attach the validated capability verdict and applied threshold to its decision record."""
+    """Attach validated forecasts and their applied policy to the routing decision."""
+    if outcome.llm_v2_forecast is not None:
+        return _with_llm_v2_forecast(decision, outcome.llm_v2_forecast)
     forecast: Final = outcome.capability_forecast
     if forecast is None:
         return decision
@@ -1320,12 +1346,12 @@ class ComplexityRouter(CustomLogger):
         capability_config: Final = self.config.capability_classifier_config
         self._classifier_response_format: Mapping[str, object] | None = (
             (
-                llm_v2_response_format(self.config.llm_v2_config.response_format)
-                if self.config.llm_v2_config is not None
-                else capability_classifier_response_format(
+                capability_classifier_response_format(
                     capability_config.response_format if capability_config is not None else "json_schema"
                 )
                 if self.config.classifier_type == "capability"
+                else llm_v2_response_format(self.config.llm_v2_config.response_format)
+                if self.config.llm_v2_config is not None
                 else type_to_response_format_param(_tier_classification_model(self.config.classifier_wire_labels()))
             )
             if llm_classifier_configured
@@ -1353,15 +1379,15 @@ class ComplexityRouter(CustomLogger):
         llm_config: Final = self.config.classifier_llm_config
         if llm_config is None:
             raise ValueError("classifier_llm_config is not set")
-        v2: Final = self.config.llm_v2_config
-        if v2 is not None:
-            pools: Final = self._tier_pools()
-            return v2.system_prompt(pools[v2.efficient_tier][0], pools[v2.capable_tier][0])
         if self.config.classifier_type == "capability":
             capability: Final = self.config.capability_classifier_config
             return capability_classifier_system_prompt(
                 capability.response_format if capability is not None else "json_schema"
             )
+        v2: Final = self.config.llm_v2_config
+        if v2 is not None:
+            pools: Final = self._tier_pools()
+            return v2.system_prompt(pools[v2.efficient_tier][0], pools[v2.capable_tier][0])
         definitions: Final = self.config.tier_definitions
         if definitions is not None:
             return custom_tier_classification_prompt(
@@ -2001,7 +2027,9 @@ class ComplexityRouter(CustomLogger):
         except Exception as e:  # noqa: BLE001 -- external LLM call can fail in many distinct ways (timeout, provider error, validation, parse error); any failure must fall back to the configured fallback path
             if breaker is not None and permit is not None:
                 breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e))
-            return self._classifier_failure_outcome(f"LLM classifier failed ({e})", prompt, system_prompt, scored)
+            return self._classifier_failure_outcome(
+                f"LLM classifier failed ({type(e).__name__})", prompt, system_prompt, scored
+            )
 
     def _classifier_failure_outcome(
         self,
@@ -2140,6 +2168,20 @@ class ComplexityRouter(CustomLogger):
             tier=tier, score=None, signals=("classifier-failed:default-model",), cause="default_model_fallback"
         )
 
+    def _classifier_caller_constraints(
+        self, system_prompt: str | None, request_kwargs: Mapping[str, object] | None
+    ) -> str | None:
+        """Exclude Claude Code's environment and skill catalogs from task forecasts."""
+        return (
+            None
+            if any(
+                is_claude_code_user_agent(user_agent)
+                for metadata in (self._iter_metadata_dicts(request_kwargs) if request_kwargs is not None else ())
+                if isinstance(user_agent := metadata.get("user_agent"), str)
+            )
+            else system_prompt
+        )
+
     async def _classify_with_llm(
         self,
         prompt: str,
@@ -2189,15 +2231,7 @@ class ComplexityRouter(CustomLogger):
         )
 
         encrypted_task: Final = _encrypted_classifier_task(request_kwargs, marker_pairs)
-        caller_system_prompt: Final = (
-            None
-            if any(
-                is_claude_code_user_agent(user_agent)
-                for metadata in (self._iter_metadata_dicts(request_kwargs) if request_kwargs is not None else ())
-                if isinstance(user_agent := metadata.get("user_agent"), str)
-            )
-            else system_prompt
-        )
+        caller_system_prompt: Final = self._classifier_caller_constraints(system_prompt, request_kwargs)
         user_payload: Final = self._build_classifier_user_payload(
             prompt="The delegated task in the following agent_message." if encrypted_task is not None else prompt,
             system_prompt=caller_system_prompt,
@@ -2229,57 +2263,6 @@ class ComplexityRouter(CustomLogger):
             raise ValueError(f"LLM classifier returned an unrecognized tier: {raw_tier!r}")
         return tier, classifier_cost
 
-    async def _classify_with_llm_v2(
-        self,
-        prompt: str,
-        system_prompt: str | None,
-        request_kwargs: Mapping[str, object] | None,
-        messages: Sequence[Mapping[str, object]] | None,
-    ) -> ClassificationOutcome:
-        v2: Final = self.config.llm_v2_config
-        if v2 is None or self._classifier_system_prompt is None:
-            raise ValueError("llm_v2_config is not set")
-        request: Final[Mapping[str, object]] = request_kwargs or MappingProxyType({})
-        markers: Final = self._reminder_markers_for_request(request)
-        asks: Final = tuple(reversed(tuple(_iter_human_asks_newest_first(messages or (), markers))))
-        encrypted: Final = _encrypted_classifier_task(request_kwargs, markers)
-        task_context: Final[LLMV2TaskContext] = {
-            "caller_constraints": system_prompt,
-            "task_and_follow_ups": asks or (prompt,),
-        }
-        task: Final = json.dumps(task_context)
-        image_parts: Final = self._classifier_image_parts(messages)
-        text_part: Final[ChatCompletionTextObject] = {"type": "text", "text": task}
-        user_content: Final[str | Sequence[ChatCompletionTextObject | ChatCompletionImageObject]] = (
-            [text_part, *image_parts] if image_parts else task  # mutable-ok: provider adapters require content arrays
-        )
-        system_message: Final[ChatCompletionSystemMessage] = {
-            "role": "system",
-            "content": self._classifier_system_prompt,
-        }
-        user_message: Final[ChatCompletionUserMessage] = {"role": "user", "content": user_content}
-        messages_for_call: Final[list[AllMessageValues]] = [  # mutable-ok: Router requires an SDK message list
-            system_message,
-            user_message,
-        ]
-        content, classifier_cost = await self._call_classifier_model(
-            messages_for_call, request_kwargs, encrypted_task=encrypted, max_output_tokens=v2.max_output_tokens
-        )
-        try:
-            verdict: Final = LLMV2Verdict.model_validate_json(content)
-        except ValidationError:
-            return self._classifier_failure_outcome("Invalid LLM V2 forecast", prompt, system_prompt)._replace(
-                classifier_cost=classifier_cost
-            )
-        decision: Final = v2.classify(verdict)
-        return ClassificationOutcome(
-            tier=ComplexityTier(v2.efficient_tier if decision.use_efficient else v2.capable_tier),
-            score=None,
-            signals=decision.signals,
-            cause="llm_v2_classifier",
-            classifier_cost=classifier_cost,
-        )
-
     async def _classify_with_capability_llm(
         self,
         prompt: str,
@@ -2347,6 +2330,62 @@ class ComplexityRouter(CustomLogger):
         )
         return ComplexityTier(selected_tier), classifier_cost, forecast
 
+    async def _classify_with_llm_v2(
+        self,
+        prompt: str,
+        system_prompt: str | None,
+        request_kwargs: Mapping[str, object] | None,
+        messages: Sequence[Mapping[str, object]] | None,
+    ) -> ClassificationOutcome:
+        v2: Final = self.config.llm_v2_config
+        if v2 is None or self._classifier_system_prompt is None:
+            raise ValueError("llm_v2_config is not set")
+        request: Final[Mapping[str, object]] = request_kwargs or MappingProxyType({})
+        markers: Final = self._reminder_markers_for_request(request)
+        encrypted: Final = _encrypted_classifier_task(request_kwargs, markers)
+        asks: Final = (
+            ("The delegated task in the following agent_message.",)
+            if encrypted is not None
+            else tuple(reversed(tuple(_iter_human_asks_newest_first(messages or (), markers))))
+        )
+        task_context: Final[LLMV2TaskContext] = {
+            "caller_constraints": self._classifier_caller_constraints(system_prompt, request_kwargs),
+            "task_and_follow_ups": asks or (prompt,),
+        }
+        task: Final = json.dumps(task_context)
+        image_parts: Final = self._classifier_image_parts(messages)
+        text_part: Final[ChatCompletionTextObject] = {"type": "text", "text": task}
+        user_content: Final[str | Sequence[ChatCompletionTextObject | ChatCompletionImageObject]] = (
+            [text_part, *image_parts] if image_parts else task  # mutable-ok: provider adapters require content arrays
+        )
+        system_message: Final[ChatCompletionSystemMessage] = {
+            "role": "system",
+            "content": self._classifier_system_prompt,
+        }
+        user_message: Final[ChatCompletionUserMessage] = {"role": "user", "content": user_content}
+        messages_for_call: Final[list[AllMessageValues]] = [  # mutable-ok: Router requires an SDK message list
+            system_message,
+            user_message,
+        ]
+        content, classifier_cost = await self._call_classifier_model(
+            messages_for_call, request_kwargs, encrypted_task=encrypted, max_output_tokens=v2.max_output_tokens
+        )
+        try:
+            verdict: Final = LLMV2Verdict.model_validate_json(unwrap_classifier_json(content))
+        except ValidationError:
+            return self._classifier_failure_outcome("Invalid LLM V2 forecast", prompt, system_prompt)._replace(
+                classifier_cost=classifier_cost
+            )
+        decision: Final = v2.classify(verdict)
+        return ClassificationOutcome(
+            tier=ComplexityTier(v2.efficient_tier if decision.use_efficient else v2.capable_tier),
+            score=None,
+            signals=decision.signals,
+            cause="llm_v2_classifier",
+            classifier_cost=classifier_cost,
+            llm_v2_forecast=decision,
+        )
+
     async def _call_classifier_model(
         self,
         messages_for_call: list[AllMessageValues],  # mutable-ok: SDKs require a list
@@ -4470,5 +4509,5 @@ class ComplexityRouter(CustomLogger):
             model=routed_model,
             messages=messages if has_original_messages else None,
             litellm_params=tier_litellm_params,
-            routing_decision=_with_capability_forecast(routing_decision, outcome),
+            routing_decision=_with_classifier_forecast(routing_decision, outcome),
         )

litellm/router_strategy/complexity_router/config.py

@@ -970,9 +970,8 @@ class ComplexityRouterConfig(BaseModel):
         default="heuristic",
         description=(
             "Classification strategy: local regex/keyword scoring, the bundled trained four-tier heuristic, "
-            "an LLM tier-selection call, a Switchyard-compatible capability forecast, a joint task-demand and "
-            "capability forecast, a custom classifier plugin, 'heuristic_first', which scores locally and only "
-            "pays for the LLM classifier when the "
+            "an LLM tier-selection call, a Switchyard-compatible capability forecast, a joint Fuse V2 forecast, "
+            "a custom classifier plugin, 'heuristic_first', which scores locally and only pays for the LLM classifier when the "
             "local scorer does not confidently land a cheap tier, or 'hybrid', which trusts the local scorer "
             "everywhere except when its score lands near a tier boundary"
         ),
@@ -1594,6 +1593,8 @@ class ComplexityRouterConfig(BaseModel):
             return self
         if v2 is None:
             raise ValueError("llm_v2_config is required when classifier_type is llm_v2")
+        if self.classifier_fallback != "heuristic":
+            raise ValueError("llm_v2 always fails closed to capable_tier; classifier_fallback cannot override it")
         llm: Final = self.classifier_llm_config
         if self.adaptive or self.tier_definitions is not None or self.enable_non_reasoning_tier:
             raise ValueError("llm_v2 requires two built-in tiers and adaptive=false")

litellm/router_utils/cooldown_handlers.py

@@ -9,6 +9,7 @@ Router cooldown handlers
 import asyncio
 import math
 from collections.abc import Mapping
+from datetime import datetime
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
@@ -637,3 +638,23 @@ def cast_exception_status_to_int(exception_status: str | int) -> int:
             )
             exception_status = 500
     return exception_status
+
+
+def is_caller_timeout_408(
+    model_call_details: Mapping[str, object], exception_status: str | int, ended: datetime | None = None
+) -> bool:
+    """A 408 that arrives before the caller-set timeout could have fired came from the provider.
+
+    ``ended`` overrides ``model_call_details["end_time"]`` for callers that run before the
+    failure logger has stamped the current API call's end time."""
+    if cast_exception_status_to_int(exception_status) != 408:
+        return False
+    litellm_params: Final = model_call_details.get("litellm_params")
+    if not isinstance(litellm_params, Mapping) or not litellm_params.get("client_side_timeout"):
+        return False
+    timeout: Final = litellm_params.get("timeout")
+    started: Final = model_call_details.get("api_call_start_time") or model_call_details.get("start_time")
+    finished: Final = ended if ended is not None else model_call_details.get("end_time")
+    if not isinstance(timeout, (int, float)) or not isinstance(started, datetime) or not isinstance(finished, datetime):
+        return False
+    return (finished - started).total_seconds() >= timeout

litellm/router_utils/fallback_event_handlers.py

@@ -2,7 +2,9 @@ import hashlib
 import json
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
+from datetime import datetime
 from enum import Enum
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
 import litellm
@@ -20,6 +22,7 @@ from litellm.router_utils.cooldown_handlers import (
     _set_cooldown_deployments,  # pyright: ignore[reportPrivateUsage] - shared helper, used across router_utils
     cast_exception_status_to_int,
     is_advisor_orchestration_failure,
+    is_caller_timeout_408,
 )
 from litellm.router_utils.router_callbacks.track_deployment_metrics import (
     increment_deployment_failures_for_current_minute,
@@ -36,12 +39,14 @@ else:
 # Status codes a generic API call's caller-supplied resource id can trigger on its own
 # (e.g. a nonexistent file/batch/thread id), independent of the selected deployment's health.
 _REQUEST_SCOPED_STATUS_CODES: Final = frozenset((404,))
+_NO_MODEL_CALL_DETAILS: Final[Mapping[str, object]] = MappingProxyType({})
 
 
 def _trigger_cooldown_for_failed_deployment(
     litellm_router: LitellmRouter,
     kwargs: Mapping[str, object],
     exception: Exception,
+    model_call_details: Mapping[str, object] = _NO_MODEL_CALL_DETAILS,
 ) -> None:
     """
     Trigger cooldown for a failed fallback deployment.
@@ -80,7 +85,11 @@ def _trigger_cooldown_for_failed_deployment(
         # timeout, which litellm.Timeout reports as status 408 regardless of the deployment's
         # actual health. Left unguarded, a caller could force a 408 on every deployment in
         # the fallback chain from a single request with a near-zero timeout.
-        if kwargs.get("client_side_timeout") and cast_exception_status_to_int(exception_status) == 408:
+        if is_caller_timeout_408(
+            model_call_details,
+            exception_status,
+            ended=datetime.now(),  # noqa: DTZ005  # naive to match the logging pipeline's api_call_start_time
+        ):
             verbose_router_logger.debug(
                 "Not triggering cooldown for fallback deployment: a caller-supplied "
                 "x-litellm-timeout caused this 408, not deployment health."
@@ -579,6 +588,7 @@ async def run_async_fallback(
                     litellm_router=litellm_router,
                     kwargs=kwargs,
                     exception=e,
+                    model_call_details=logging_obj.model_call_details,
                 )
     raise error_from_fallbacks
 

litellm/types/utils.py

@@ -2990,6 +2990,12 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
     classifier_p_solve: float  # writable-ok: added only when a capability verdict is available
     classifier_calibrated_p_solve: ReadOnly[float]
     classifier_calibration_version: ReadOnly[str]
+    classifier_efficient_p_solve: ReadOnly[float]
+    classifier_capable_p_solve: ReadOnly[float]
+    classifier_calibrated_efficient_p_solve: ReadOnly[float]
+    classifier_calibrated_capable_p_solve: ReadOnly[float]
+    classifier_max_quality_gap: ReadOnly[float]
+    classifier_prompt_version: ReadOnly[str]
     classifier_threshold: float  # writable-ok: added only when a capability verdict is available
     escalated: bool
     context_escalated: bool  # writable-ok: Pydantic warns on ReadOnly TypedDict fields
@@ -3026,6 +3032,12 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
         "classifier_p_solve",
         "classifier_calibrated_p_solve",
         "classifier_calibration_version",
+        "classifier_efficient_p_solve",
+        "classifier_capable_p_solve",
+        "classifier_calibrated_efficient_p_solve",
+        "classifier_calibrated_capable_p_solve",
+        "classifier_max_quality_gap",
+        "classifier_prompt_version",
         "classifier_threshold",
         "escalated",
         "context_escalated",

schema.prisma

@@ -17,6 +17,7 @@ model LiteLLM_BudgetTable {
   max_parallel_requests Int?
   tpm_limit     BigInt?
   rpm_limit     BigInt?
+  tpd_limit     BigInt?
   model_max_budget Json?
   budget_duration String?
   budget_reset_at DateTime?
@@ -133,6 +134,7 @@ model LiteLLM_TeamTable {
     max_parallel_requests Int?
     tpm_limit     BigInt?
     rpm_limit     BigInt?
+    tpd_limit     BigInt?
     budget_duration String? 
     budget_reset_at DateTime?
     blocked Boolean @default(false)
@@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable {
     max_parallel_requests   Int?
     tpm_limit               BigInt?
     rpm_limit               BigInt?
+    tpd_limit               BigInt?
     budget_duration         String?
     budget_reset_at         DateTime?
     blocked                 Boolean  @default(false)
@@ -438,6 +441,7 @@ model LiteLLM_VerificationToken {
     blocked Boolean?
     tpm_limit     BigInt?
     rpm_limit     BigInt?
+    tpd_limit     BigInt?
     max_budget Float?    
     budget_duration String? 
     budget_reset_at DateTime?
@@ -534,6 +538,7 @@ model LiteLLM_DeletedVerificationToken {
     blocked                 Boolean?
     tpm_limit               BigInt?
     rpm_limit               BigInt?
+    tpd_limit               BigInt?
     max_budget              Float?
     budget_duration         String?
     budget_reset_at         DateTime?

terraform/provider/CHANGELOG.md

@@ -16,6 +16,7 @@ longer signal it.
 
 ### Added
 
+- **team_member_add**: `tpm_limit`, `rpm_limit`, `budget_duration`, and `allowed_models` attributes on `litellm_team_member_add`, applied to every member of the resource; `budget_duration` and `allowed_models` ride on `/team/member_add`, while the limits are sent through `/team/member_update`, which is where the proxy accepts them
 - **team**: Optional `team_id` argument on `litellm_team`, so teams can be created with a stable, human-readable ID instead of a provider-generated UUID; changing it forces replacement
 - **jwt_key_mapping**: New `litellm_jwt_key_mapping` resource for the proxy's JWT to virtual key mappings, so JWT clients identified by a claim (`client_id`, `azp`, `sub`) map to virtual keys and inherit their models, budgets and rate limits. Supports `description` and `is_active`, rotating the mapped key in place, and forces replacement when the claim name or value changes
 - **team**: `soft_budget`, `tags`, and `soft_budget_alerting_emails` attributes on `litellm_team`, matching what `/team/new` and `/team/update` already accept; `soft_budget_alerting_emails` is sent under `metadata`, where the proxy reads it

terraform/provider/docs/resources/team_member_add.md

@@ -27,6 +27,10 @@ resource "litellm_team_member_add" "example" {
   }
 
   max_budget_in_team = 100.0
+  budget_duration    = "30d"
+  tpm_limit          = 100000
+  rpm_limit          = 100
+  allowed_models     = ["gpt-4"]
 }
 ```
 
@@ -152,6 +156,12 @@ resource "litellm_team_member_add" "budget_example" {
   * `user_email` - (Optional) The email of the user to add to the team.
   * `role` - (Required) The role of the user in the team. Must be one of: "admin" or "user".
 * `max_budget_in_team` - (Optional) The maximum budget allocated for the team members.
+* `budget_duration` - (Optional) Duration after which each member's budget resets, for example "1h", "24h", "7d", "30d". If not set, the budget never resets.
+* `tpm_limit` - (Optional) Tokens per minute limit applied to each team member. Sent via `/team/member_update` after members are added, since `/team/member_add` does not accept it.
+* `rpm_limit` - (Optional) Requests per minute limit applied to each team member. Sent via `/team/member_update` after members are added, since `/team/member_add` does not accept it.
+* `allowed_models` - (Optional) List of models each team member can access. If not set, members inherit the team's `default_team_member_models` or all team models.
+
+Removing `budget_duration`, `tpm_limit`, `rpm_limit`, or `allowed_models` from the configuration clears that setting on every member through `/team/member_update`.
 
 ## Import
 

terraform/provider/litellm/resource_team_member_add.go

@@ -49,10 +49,105 @@ func resourceLiteLLMTeamMemberAdd() *schema.Resource {
 				Type:     schema.TypeFloat,
 				Optional: true,
 			},
+			"tpm_limit": {
+				Type:     schema.TypeInt,
+				Optional: true,
+			},
+			"rpm_limit": {
+				Type:     schema.TypeInt,
+				Optional: true,
+			},
+			"budget_duration": {
+				Type:     schema.TypeString,
+				Optional: true,
+			},
+			"allowed_models": {
+				Type:     schema.TypeList,
+				Optional: true,
+				Elem:     &schema.Schema{Type: schema.TypeString},
+			},
 		},
 	}
 }
 
+func expandAllowedModels(raw []interface{}) []string {
+	models := make([]string, 0, len(raw))
+	for _, m := range raw {
+		models = append(models, m.(string))
+	}
+	return models
+}
+
+func applyAddOnlySettings(d *schema.ResourceData, payload map[string]interface{}) {
+	if v, ok := d.GetOk("budget_duration"); ok {
+		payload["budget_duration"] = v.(string)
+	}
+	if v, ok := d.GetOk("allowed_models"); ok {
+		payload["allowed_models"] = expandAllowedModels(v.([]interface{}))
+	}
+}
+
+func applyLimits(d *schema.ResourceData, payload map[string]interface{}) {
+	for _, key := range []string{"tpm_limit", "rpm_limit"} {
+		if v, ok := d.GetOk(key); ok {
+			payload[key] = v.(int)
+		}
+	}
+}
+
+func applyUpdateSettings(d *schema.ResourceData, payload map[string]interface{}) {
+	applyAddOnlySettings(d, payload)
+	applyLimits(d, payload)
+	for _, key := range []string{"tpm_limit", "rpm_limit", "budget_duration"} {
+		if _, ok := d.GetOk(key); !ok && d.HasChange(key) {
+			payload[key] = nil
+		}
+	}
+	if _, ok := d.GetOk("allowed_models"); !ok && d.HasChange("allowed_models") {
+		payload["allowed_models"] = []string{}
+	}
+}
+
+func memberIdentity(member map[string]interface{}, payload map[string]interface{}) {
+	if userID, ok := member["user_id"].(string); ok && userID != "" {
+		payload["user_id"] = userID
+	}
+	if userEmail, ok := member["user_email"].(string); ok && userEmail != "" {
+		payload["user_email"] = userEmail
+	}
+}
+
+// tpm/rpm limits are only accepted by /team/member_update, not /team/member_add
+func setMemberLimits(client *Client, d *schema.ResourceData, teamID string, members []map[string]interface{}) error {
+	limits := map[string]interface{}{}
+	applyLimits(d, limits)
+	if len(limits) == 0 {
+		return nil
+	}
+	for _, member := range members {
+		updateData := map[string]interface{}{
+			"team_id": teamID,
+		}
+		for k, v := range limits {
+			updateData[k] = v
+		}
+		memberIdentity(member, updateData)
+
+		log.Printf("[DEBUG] Set team member limits request payload: %+v", updateData)
+
+		resp, err := MakeRequest(client, "POST", "/team/member_update", updateData)
+		if err != nil {
+			return fmt.Errorf("error setting team member limits: %v", err)
+		}
+		defer resp.Body.Close()
+
+		if err := handleResponse(resp, "setting team member limits"); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func resourceLiteLLMTeamMemberAddCreate(d *schema.ResourceData, m interface{}) error {
 	client := m.(*Client)
 
@@ -81,6 +176,7 @@ func resourceLiteLLMTeamMemberAddCreate(d *schema.ResourceData, m interface{}) e
 		"team_id":            teamID,
 		"max_budget_in_team": maxBudget,
 	}
+	applyAddOnlySettings(d, memberData)
 
 	log.Printf("[DEBUG] Create team members request payload: %+v", memberData)
 
@@ -94,9 +190,12 @@ func resourceLiteLLMTeamMemberAddCreate(d *schema.ResourceData, m interface{}) e
 		return err
 	}
 
-	// Set ID as team_id since this resource manages all members for a team
 	d.SetId(teamID)
 
+	if err := setMemberLimits(client, d, teamID, membersList); err != nil {
+		return err
+	}
+
 	return resourceLiteLLMTeamMemberAddRead(d, m)
 }
 
@@ -140,11 +239,13 @@ func resourceLiteLLMTeamMemberAddUpdate(d *schema.ResourceData, m interface{}) e
 	// Track which members have been updated to avoid duplicates
 	updatedMembers := make(map[string]bool)
 
-	// Check if max_budget_in_team has changed
-	if d.HasChange("max_budget_in_team") {
-		log.Printf("[DEBUG] max_budget_in_team changed, updating all existing members with new budget: %f", maxBudget)
+	// Check if any team-wide member setting has changed
+	settingsChanged := d.HasChange("max_budget_in_team") || d.HasChange("tpm_limit") || d.HasChange("rpm_limit") ||
+		d.HasChange("budget_duration") || d.HasChange("allowed_models")
+	if settingsChanged {
+		log.Printf("[DEBUG] Member settings changed, updating all existing members")
 
-		// Update ALL existing members with the new budget
+		// Update ALL existing members with the new settings
 		for key, newMember := range newMemberMap {
 			if _, exists := oldMemberMap[key]; exists {
 				updateData := map[string]interface{}{
@@ -152,22 +253,18 @@ func resourceLiteLLMTeamMemberAddUpdate(d *schema.ResourceData, m interface{}) e
 					"role":               newMember["role"].(string),
 					"max_budget_in_team": maxBudget,
 				}
-				if userID, ok := newMember["user_id"].(string); ok && userID != "" {
-					updateData["user_id"] = userID
-				}
-				if userEmail, ok := newMember["user_email"].(string); ok && userEmail != "" {
-					updateData["user_email"] = userEmail
-				}
+				applyUpdateSettings(d, updateData)
+				memberIdentity(newMember, updateData)
 
-				log.Printf("[DEBUG] Update team member budget request payload: %+v", updateData)
+				log.Printf("[DEBUG] Update team member settings request payload: %+v", updateData)
 
 				resp, err := MakeRequest(client, "POST", "/team/member_update", updateData)
 				if err != nil {
-					return fmt.Errorf("error updating team member budget: %v", err)
+					return fmt.Errorf("error updating team member settings: %v", err)
 				}
 				defer resp.Body.Close()
 
-				if err := handleResponse(resp, "updating team member budget"); err != nil {
+				if err := handleResponse(resp, "updating team member settings"); err != nil {
 					return err
 				}
 
@@ -220,12 +317,8 @@ func resourceLiteLLMTeamMemberAddUpdate(d *schema.ResourceData, m interface{}) e
 					"role":               newMember["role"].(string),
 					"max_budget_in_team": maxBudget,
 				}
-				if userID, ok := newMember["user_id"].(string); ok && userID != "" {
-					updateData["user_id"] = userID
-				}
-				if userEmail, ok := newMember["user_email"].(string); ok && userEmail != "" {
-					updateData["user_email"] = userEmail
-				}
+				applyUpdateSettings(d, updateData)
+				memberIdentity(newMember, updateData)
 
 				log.Printf("[DEBUG] Update team member request payload: %+v", updateData)
 
@@ -265,6 +358,7 @@ func resourceLiteLLMTeamMemberAddUpdate(d *schema.ResourceData, m interface{}) e
 			"team_id":            teamID,
 			"max_budget_in_team": maxBudget,
 		}
+		applyAddOnlySettings(d, memberData)
 
 		log.Printf("[DEBUG] Adding new team members request payload: %+v", memberData)
 
@@ -277,6 +371,10 @@ func resourceLiteLLMTeamMemberAddUpdate(d *schema.ResourceData, m interface{}) e
 		if err := handleResponse(resp, "adding team members"); err != nil {
 			return err
 		}
+
+		if err := setMemberLimits(client, d, teamID, membersToAdd); err != nil {
+			return err
+		}
 	}
 
 	return resourceLiteLLMTeamMemberAddRead(d, m)

terraform/provider/litellm/resource_team_member_add_test.go

@@ -0,0 +1,274 @@
+package litellm
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
+)
+
+func TestTeamMemberAddCreateSendsMemberSettings(t *testing.T) {
+	var addPayload map[string]interface{}
+	var updatePayloads []map[string]interface{}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		var payload map[string]interface{}
+		json.Unmarshal(body, &payload)
+		switch r.URL.Path {
+		case "/team/member_add":
+			addPayload = payload
+		case "/team/member_update":
+			updatePayloads = append(updatePayloads, payload)
+		default:
+			t.Errorf("unexpected request path: %s", r.URL.Path)
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	client := NewClient(srv.URL, "test-key", true)
+	d := schema.TestResourceDataRaw(t, resourceLiteLLMTeamMemberAdd().Schema, map[string]interface{}{
+		"team_id": "team-1",
+		"member": []interface{}{
+			map[string]interface{}{
+				"user_id": "user-1",
+				"role":    "user",
+			},
+		},
+		"max_budget_in_team": 25.0,
+		"tpm_limit":          1000,
+		"rpm_limit":          10,
+		"budget_duration":    "30d",
+		"allowed_models":     []interface{}{"claude-opus-4-6-v1"},
+	})
+
+	if err := resourceLiteLLMTeamMemberAddCreate(d, client); err != nil {
+		t.Fatalf("create failed: %v", err)
+	}
+
+	if addPayload["budget_duration"] != "30d" {
+		t.Fatalf("member_add payload sent budget_duration %v, want 30d", addPayload["budget_duration"])
+	}
+	wantModels := []interface{}{"claude-opus-4-6-v1"}
+	if !reflect.DeepEqual(addPayload["allowed_models"], wantModels) {
+		t.Fatalf("member_add payload sent allowed_models %v, want %v", addPayload["allowed_models"], wantModels)
+	}
+	if _, ok := addPayload["tpm_limit"]; ok {
+		t.Fatalf("member_add payload must not carry tpm_limit, got %v", addPayload["tpm_limit"])
+	}
+
+	if len(updatePayloads) != 1 {
+		t.Fatalf("expected 1 member_update call for limits, got %d", len(updatePayloads))
+	}
+	update := updatePayloads[0]
+	if update["tpm_limit"] != float64(1000) {
+		t.Fatalf("member_update payload sent tpm_limit %v, want 1000", update["tpm_limit"])
+	}
+	if update["rpm_limit"] != float64(10) {
+		t.Fatalf("member_update payload sent rpm_limit %v, want 10", update["rpm_limit"])
+	}
+	if update["user_id"] != "user-1" {
+		t.Fatalf("member_update payload sent user_id %v, want user-1", update["user_id"])
+	}
+}
+
+func TestTeamMemberAddCreateOmitsUnsetSettings(t *testing.T) {
+	var addPayload map[string]interface{}
+	updateCalls := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		switch r.URL.Path {
+		case "/team/member_add":
+			json.Unmarshal(body, &addPayload)
+		case "/team/member_update":
+			updateCalls++
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	client := NewClient(srv.URL, "test-key", true)
+	d := schema.TestResourceDataRaw(t, resourceLiteLLMTeamMemberAdd().Schema, map[string]interface{}{
+		"team_id": "team-1",
+		"member": []interface{}{
+			map[string]interface{}{
+				"user_id": "user-1",
+				"role":    "user",
+			},
+		},
+	})
+
+	if err := resourceLiteLLMTeamMemberAddCreate(d, client); err != nil {
+		t.Fatalf("create failed: %v", err)
+	}
+
+	for _, field := range []string{"tpm_limit", "rpm_limit", "budget_duration", "allowed_models"} {
+		if _, ok := addPayload[field]; ok {
+			t.Fatalf("member_add payload must not carry unset %s, got %v", field, addPayload[field])
+		}
+	}
+	if updateCalls != 0 {
+		t.Fatalf("expected no member_update calls without limits, got %d", updateCalls)
+	}
+}
+
+func TestTeamMemberAddCreateSetsIDBeforeLimitsFail(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if r.URL.Path == "/team/member_update" {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(`{"error":"boom"}`))
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	client := NewClient(srv.URL, "test-key", true)
+	d := schema.TestResourceDataRaw(t, resourceLiteLLMTeamMemberAdd().Schema, map[string]interface{}{
+		"team_id": "team-1",
+		"member": []interface{}{
+			map[string]interface{}{
+				"user_id": "user-1",
+				"role":    "user",
+			},
+		},
+		"tpm_limit": 1000,
+	})
+
+	if err := resourceLiteLLMTeamMemberAddCreate(d, client); err == nil {
+		t.Fatal("create should fail when member_update fails")
+	}
+	if d.Id() != "team-1" {
+		t.Fatalf("resource ID = %q after failed limits call, want team-1 so Terraform can taint and recreate it", d.Id())
+	}
+}
+
+// newTeamMemberUpdateResourceData builds a ResourceData with one member in state
+// and a real old -> new diff on the scalar settings, so d.HasChange and d.GetOk
+// behave as they do during a real Update call
+func newTeamMemberUpdateResourceData(t *testing.T, old, new map[string]string) *schema.ResourceData {
+	t.Helper()
+	attrs := map[string]string{
+		"team_id":             "team-1",
+		"member.#":            "1",
+		"member.1.user_id":    "user-1",
+		"member.1.user_email": "",
+		"member.1.role":       "user",
+		"allowed_models.#":    "0",
+		"max_budget_in_team":  "25",
+	}
+	for k, v := range old {
+		attrs[k] = v
+	}
+	diffAttrs := map[string]*terraform.ResourceAttrDiff{}
+	for k, v := range new {
+		diffAttrs[k] = &terraform.ResourceAttrDiff{Old: attrs[k], New: v}
+	}
+	for k := range old {
+		if _, ok := new[k]; !ok {
+			diffAttrs[k] = &terraform.ResourceAttrDiff{Old: attrs[k], New: "", NewRemoved: true}
+		}
+	}
+	state := &terraform.InstanceState{ID: "team-1", Attributes: attrs}
+	d, err := schema.InternalMap(resourceLiteLLMTeamMemberAdd().Schema).Data(state, &terraform.InstanceDiff{Attributes: diffAttrs})
+	if err != nil {
+		t.Fatalf("building ResourceData returned error: %v", err)
+	}
+	return d
+}
+
+func runTeamMemberUpdate(t *testing.T, d *schema.ResourceData) []map[string]interface{} {
+	t.Helper()
+	var updatePayloads []map[string]interface{}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/team/member_update" {
+			t.Errorf("unexpected request path: %s", r.URL.Path)
+		}
+		body, _ := io.ReadAll(r.Body)
+		var payload map[string]interface{}
+		json.Unmarshal(body, &payload)
+		updatePayloads = append(updatePayloads, payload)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`{}`))
+	}))
+	defer srv.Close()
+
+	if err := resourceLiteLLMTeamMemberAddUpdate(d, NewClient(srv.URL, "test-key", true)); err != nil {
+		t.Fatalf("update failed: %v", err)
+	}
+	if len(updatePayloads) != 1 {
+		t.Fatalf("expected 1 member_update call, got %d", len(updatePayloads))
+	}
+	return updatePayloads
+}
+
+func TestTeamMemberAddUpdateSendsChangedSettings(t *testing.T) {
+	d := newTeamMemberUpdateResourceData(t,
+		map[string]string{"tpm_limit": "1000", "rpm_limit": "10", "budget_duration": "30d"},
+		map[string]string{"tpm_limit": "500", "rpm_limit": "5", "budget_duration": "7d", "allowed_models.#": "1", "allowed_models.0": "gpt-5.2"},
+	)
+
+	update := runTeamMemberUpdate(t, d)[0]
+	if update["tpm_limit"] != float64(500) || update["rpm_limit"] != float64(5) {
+		t.Fatalf("member_update payload limits = %v/%v, want 500/5", update["tpm_limit"], update["rpm_limit"])
+	}
+	if update["budget_duration"] != "7d" {
+		t.Fatalf("member_update payload budget_duration = %v, want 7d", update["budget_duration"])
+	}
+	if !reflect.DeepEqual(update["allowed_models"], []interface{}{"gpt-5.2"}) {
+		t.Fatalf("member_update payload allowed_models = %v, want [gpt-5.2]", update["allowed_models"])
+	}
+	if update["user_id"] != "user-1" {
+		t.Fatalf("member_update payload user_id = %v, want user-1", update["user_id"])
+	}
+}
+
+func TestTeamMemberAddUpdateClearsRemovedSettings(t *testing.T) {
+	d := newTeamMemberUpdateResourceData(t,
+		map[string]string{"tpm_limit": "1000", "rpm_limit": "10", "budget_duration": "30d", "allowed_models.#": "1", "allowed_models.0": "gpt-5.2"},
+		map[string]string{"allowed_models.#": "0"},
+	)
+
+	update := runTeamMemberUpdate(t, d)[0]
+	for _, field := range []string{"tpm_limit", "rpm_limit", "budget_duration"} {
+		v, present := update[field]
+		if !present {
+			t.Fatalf("member_update payload omitted removed %s, so the proxy would keep the old value", field)
+		}
+		if v != nil {
+			t.Fatalf("member_update payload %s = %v, want explicit null", field, v)
+		}
+	}
+	if !reflect.DeepEqual(update["allowed_models"], []interface{}{}) {
+		t.Fatalf("member_update payload allowed_models = %v, want empty list", update["allowed_models"])
+	}
+}
+
+func TestTeamMemberAddUpdateLeavesUnchangedSettingsAlone(t *testing.T) {
+	d := newTeamMemberUpdateResourceData(t,
+		map[string]string{"budget_duration": "30d"},
+		map[string]string{"budget_duration": "7d"},
+	)
+
+	update := runTeamMemberUpdate(t, d)[0]
+	for _, field := range []string{"tpm_limit", "rpm_limit"} {
+		if v, present := update[field]; present {
+			t.Fatalf("member_update payload must not touch never-set %s, got %v", field, v)
+		}
+	}
+	if _, present := update["allowed_models"]; present {
+		t.Fatalf("member_update payload must not touch unchanged allowed_models, got %v", update["allowed_models"])
+	}
+}

tests/local_testing/test_router_utils.py

@@ -3,6 +3,7 @@
 
 import sys, os, time
 import traceback, asyncio
+import httpx
 import pytest
 
 import litellm
@@ -402,6 +403,10 @@ def test_router_redis_cache():
 
 
 def test_router_handle_clientside_credential():
+    """A caller-supplied credential must stay scoped to the current call: it must
+    never be registered as a router deployment, or a later caller with no override
+    of their own can be load-balanced onto it and reach the provider with someone
+    else's credential (see LIT-7811)."""
     deployment = {
         "model_name": "gemini/*",
         "litellm_params": {"model": "gemini/*"},
@@ -421,7 +426,67 @@ def test_router_handle_clientside_credential():
     )
 
     assert new_deployment.litellm_params.api_key == "123"
-    assert len(router.get_model_list()) == 2
+    assert len(router.get_model_list()) == 1
+    assert router.get_deployment(model_id=new_deployment.model_info.id) is None
+
+
+async def test_router_clientside_credential_not_reused_by_other_callers(
+    respx_mock, monkeypatch: pytest.MonkeyPatch
+):
+    """End-to-end regression test for LIT-7811.
+
+    One caller's request-scoped api_key must never leak into a later, unrelated
+    caller's request. Before the fix, the router registered the caller-supplied
+    credential as a second, permanent deployment for the shared model group, so
+    plain follow-up calls with no override of their own could be load-balanced
+    onto it and reach the provider with the first caller's key.
+    """
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    route = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "id": "chatcmpl-1",
+                "object": "chat.completion",
+                "created": 0,
+                "model": "gpt-4o",
+                "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
+                "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+            },
+        )
+    )
+    router = Router(
+        model_list=[
+            {
+                "model_name": "shared-model",
+                "litellm_params": {"model": "openai/gpt-4o", "api_key": "configured-key"},
+                "model_info": {"id": "configured-deployment"},
+            }
+        ]
+    )
+
+    await router.acompletion(
+        model="shared-model",
+        messages=[{"role": "user", "content": "hi"}],
+        api_key="alternate-tenant-key",
+    )
+    assert route.calls[-1].request.headers["authorization"] == "Bearer alternate-tenant-key"
+
+    # The forwarded credential must never become a routable deployment for the
+    # model group other callers share.
+    assert [d["model_info"]["id"] for d in router.get_model_list(model_name="shared-model")] == [
+        "configured-deployment"
+    ]
+
+    for _ in range(20):
+        await router.acompletion(
+            model="shared-model",
+            messages=[{"role": "user", "content": "hi"}],
+        )
+
+    used_auth_headers = {call.request.headers["authorization"] for call in route.calls[1:]}
+    assert used_auth_headers == {"Bearer configured-key"}
 
 
 def test_router_get_async_openai_model_client():

tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json

@@ -11,7 +11,7 @@
     "user": "",
     "team_id": "",
     "organization_id": "",
-    "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"router_metadata\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}",
+    "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"router_metadata\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}",
     "cache_key": "Cache OFF",
     "spend": 0.00022500000000000002,
     "total_tokens": 30,

tests/pass_through_unit_tests/conftest.py

@@ -1,6 +1,8 @@
+import asyncio
 
 import pytest
 
+from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 
 from tests._vcr_conftest_common import (  # noqa: E402,F401
     VerboseReporterState,
@@ -45,6 +47,21 @@ def _vcr_outcome_gate(request, vcr):
     record_vcr_outcome(request, vcr)
 
 
+@pytest.fixture(autouse=True)
+async def _drain_logging_worker():
+    """
+    The logging queue is bound to the running loop, so anything left queued when a test's loop
+    goes away is carried onto the next loop and fires against that test's callbacks.
+    """
+    GLOBAL_LOGGING_WORKER.start()
+    try:
+        await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10)
+    except asyncio.TimeoutError:
+        pass
+    await GLOBAL_LOGGING_WORKER.stop()
+    yield
+
+
 def pytest_configure(config):
     _verbose_state.remember_pluginmanager(config)
     reset_vcr_diag_dir()

tests/router_unit_tests/test_router_helper_utils.py

@@ -2099,8 +2099,12 @@ def test_handle_clientside_credential_metadata_loading(
     assert result_deployment.model_info.id != "original-id-123"
     assert result_deployment.model_info.original_model_id == "original-id-123"
 
-    # Verify the deployment was added to the router
-    assert len(router.model_list) == len(model_list) + 1
+    # The caller-supplied credential must stay scoped to this call: it must never be
+    # registered as a router deployment, or a later caller with no override of their
+    # own could be load-balanced onto it and reach the provider with this credential
+    # (see LIT-7811).
+    assert len(router.model_list) == len(model_list)
+    assert router.get_deployment(model_id=result_deployment.model_info.id) is None
 
     # Test that the function correctly uses the right metadata key
     # For acompletion, it should use "metadata"
@@ -2260,14 +2264,63 @@ def test_handle_clientside_credential_with_responses_function(model_list):
     assert result_deployment.model_info.id != "original-id-responses"
     assert result_deployment.model_info.original_model_id == "original-id-responses"
 
-    # Verify the deployment was added to the router
-    assert len(router.model_list) == len(model_list) + 1
+    # The caller-supplied credential must stay scoped to this call: it must never be
+    # registered as a router deployment (see LIT-7811).
+    assert len(router.model_list) == len(model_list)
+    assert router.get_deployment(model_id=result_deployment.model_info.id) is None
 
     print(
         "✓ Success with _ageneric_api_call_with_fallbacks function name and litellm_metadata"
     )
 
 
+def test_handle_clientside_credential_still_registers_custom_pricing(model_list):
+    """A clientside-credential call must still price against the deployment's own
+    custom rate, even though the call's ephemeral deployment is never added to the
+    router (see LIT-7811): losing that registration would silently fall back to
+    public catalog pricing for every clientside-credential call on a deployment
+    with a custom rate configured."""
+    router = Router(model_list=model_list)
+    deployment = {
+        "model_name": "gpt-4.1",
+        "litellm_params": {
+            "model": "gpt-4.1",
+            "api_key": "test_key",
+            "input_cost_per_token": 0.0001234,
+            "output_cost_per_token": 0.0005678,
+        },
+        "model_info": {"id": "original-id-pricing"},
+    }
+    kwargs = {"api_key": "client_side_key", "metadata": {"model_group": "gpt-4.1"}}
+
+    result_deployment = router._handle_clientside_credential(
+        deployment=deployment, kwargs=kwargs, function_name="acompletion"
+    )
+
+    registered = litellm.model_cost.get(result_deployment.model_info.id)
+    assert registered is not None
+    assert registered["input_cost_per_token"] == 0.0001234
+    assert registered["output_cost_per_token"] == 0.0005678
+
+
+def test_register_deployment_pricing_direct_call():
+    """Direct-call unit test for the pricing-registration helper `_handle_clientside_credential`
+    relies on, so it prices a deployment that is deliberately never added to `self.model_list`."""
+    deployment = Deployment(
+        model_name="gpt-4.1",
+        litellm_params=LiteLLM_Params(
+            model="gpt-4.1",
+            api_key="test_key",
+            input_cost_per_token=0.0009999,
+        ),
+        model_info=ModelInfo(id="direct-call-pricing-id"),
+    )
+
+    Router._register_deployment_pricing(deployment=deployment)
+
+    assert litellm.model_cost["direct-call-pricing-id"]["input_cost_per_token"] == 0.0009999
+
+
 def test_get_metadata_variable_name_from_kwargs(model_list):
     """
     Test _get_metadata_variable_name_from_kwargs method returns correct metadata variable name based on kwargs content.

tests/test_litellm/a2a_protocol/test_a2a_streaming_iterator.py

@@ -100,3 +100,57 @@ async def test_custom_logger_only_never_submits_sync_success_handler(monkeypatch
 
     assert recorder.async_hook_fired is True
     assert recording_executor.submitted_for(logging_obj) == []
+
+
+class _AgentChunk:
+    def __init__(self, text: str):
+        self._text = text
+
+    def model_dump(self, mode: str, exclude_none: bool) -> dict:
+        return {"result": {"kind": "message", "role": "agent", "parts": [{"kind": "text", "text": self._text}]}}
+
+
+@pytest.mark.asyncio
+async def test_stream_completion_counts_tokens_off_the_event_loop(monkeypatch):
+    from tests.large_text import text
+    from tests.test_litellm.litellm_core_utils.event_loop_lag import (
+        assert_loop_stayed_free,
+        timed_with_loop_lags,
+        warm_tokenizer,
+    )
+
+    warm_tokenizer("gpt-5.6-luna")
+    monkeypatch.setattr(litellm, "success_callback", [])
+    monkeypatch.setattr(litellm, "_async_success_callback", [])
+    logging_obj = LitellmLogging(
+        model="a2a/test-agent",
+        messages=[{"role": "user", "content": "hi"}],
+        stream=True,
+        call_type="a2a_send_message_streaming",
+        start_time=time.time(),
+        litellm_call_id="lit-7190-test",
+        function_id="lit-7190-test",
+    )
+
+    async def _stream():
+        yield _AgentChunk(text * 100)
+
+    iterator = A2AStreamingIterator(
+        stream=_stream(),
+        request=SimpleNamespace(
+            params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": text * 100}]})
+        ),
+        logging_obj=logging_obj,
+        agent_name="test-agent",
+    )
+
+    async def drain() -> int:
+        return len([chunk async for chunk in iterator])
+
+    yielded, took, lags = await timed_with_loop_lags(drain)
+
+    assert yielded == 1
+    usage = logging_obj.model_call_details["usage"]
+    assert usage.prompt_tokens > 100_000
+    assert usage.completion_tokens > 100_000
+    assert_loop_stayed_free(took, lags)

tests/test_litellm/a2a_protocol/test_main.py

@@ -1,5 +1,7 @@
 """Tests for litellm/a2a_protocol/main.py non-streaming send behavior."""
 
+import asyncio
+
 import httpx
 import pytest
 
@@ -13,7 +15,8 @@ from a2a.compat.v0_3.types import (
 )
 
 import litellm
-from litellm.a2a_protocol.main import _send_message, _stream_messages, create_a2a_client
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.a2a_protocol.main import _send_message, _stream_messages, asend_message, create_a2a_client
 from litellm.caching.llm_caching_handler import LLMClientCache
 from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT
 from litellm.llms.custom_httpx.http_handler import (
@@ -413,3 +416,51 @@ async def test_the_pooled_a2a_client_arrives_with_cookie_persistence_disabled(is
 
     assert dict(handler.client.cookies) == {}, "the pooled A2A client kept an upstream's cookie"
     await handler.close()
+
+
+class _UsageRecorder(CustomLogger):
+    def __init__(self):
+        super().__init__()
+        self.logged = asyncio.Event()
+        self.payload = None
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        self.payload = kwargs["standard_logging_object"]
+        self.logged.set()
+
+
+@pytest.mark.asyncio
+async def test_asend_message_counts_usage_off_the_event_loop(monkeypatch):
+    from tests.large_text import text
+    from tests.test_litellm.litellm_core_utils.event_loop_lag import (
+        assert_loop_stayed_free,
+        timed_with_loop_lags,
+        warm_tokenizer,
+    )
+
+    warm_tokenizer("gpt-5.6-luna")
+    recorder = _UsageRecorder()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+    monkeypatch.setattr(litellm, "success_callback", [recorder])
+    monkeypatch.setattr(litellm, "_async_success_callback", [recorder])
+
+    reply = _conv.pb2_v10.StreamResponse()
+    reply.message.message_id = "reply-1"
+    reply.message.role = _conv.pb2_v10.Role.ROLE_AGENT
+    reply.message.parts.add().text = text * 100
+    request = SendMessageRequest(
+        id="r1",
+        params=MessageSendParams(
+            message={"messageId": "m1", "role": "user", "parts": [{"kind": "text", "text": text * 100}]}
+        ),
+    )
+
+    response, took, lags = await timed_with_loop_lags(
+        lambda: asend_message(a2a_client=_FakeClient(reply), request=request)
+    )
+
+    assert response.id == "r1"
+    await asyncio.wait_for(recorder.logged.wait(), timeout=10)
+    assert recorder.payload["prompt_tokens"] > 100_000
+    assert recorder.payload["completion_tokens"] > 100_000
+    assert_loop_stayed_free(took, lags)

tests/test_litellm/caching/test_qdrant_semantic_cache.py

@@ -1026,3 +1026,36 @@ def test_qdrant_semantic_cache_defaults_embedding_timeout():
     cache = QdrantSemanticCache.__new__(QdrantSemanticCache)
     assert cache.embedding_timeout == SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
     assert SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS < 60
+
+
+@pytest.mark.asyncio
+async def test_qdrant_async_embedding_truncates_off_the_event_loop(monkeypatch):
+    from tests.large_text import text
+    from tests.test_litellm.litellm_core_utils.event_loop_lag import (
+        assert_loop_stayed_free,
+        timed_with_loop_lags,
+        warm_tokenizer,
+    )
+
+    from litellm.caching.qdrant_semantic_cache import QdrantSemanticCache
+
+    warm_tokenizer("sem-embed")
+    cache = QdrantSemanticCache.__new__(QdrantSemanticCache)
+    cache.embedding_model = "sem-embed"
+    cache.embedding_max_input_tokens = 5
+    cache.embedding_timeout = 5
+
+    router = MagicMock()
+    router.get_configured_token_limits.return_value = (8191, None)
+    router.aembedding = AsyncMock(return_value={"data": [{"embedding": [0.1, 0.2]}]})
+    monkeypatch.setitem(
+        sys.modules,
+        "litellm.proxy.proxy_server",
+        _router_proxy_module(router, "sem-embed"),
+    )
+
+    response, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(text * 100))
+
+    assert response["data"][0]["embedding"] == [0.1, 0.2]
+    assert _token_count("sem-embed", router.aembedding.call_args.kwargs["input"]) == 5
+    assert_loop_stayed_free(took, lags)

tests/test_litellm/caching/test_redis_semantic_cache.py

@@ -1387,3 +1387,32 @@ def test_redis_semantic_cache_defaults_embedding_timeout():
     cache = RedisSemanticCache.__new__(RedisSemanticCache)
     assert cache.embedding_timeout == SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
     assert SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS < 60
+
+
+@pytest.mark.asyncio
+async def test_redis_async_embedding_truncates_off_the_event_loop(monkeypatch):
+    from tests.large_text import text
+    from tests.test_litellm.litellm_core_utils.event_loop_lag import (
+        assert_loop_stayed_free,
+        timed_with_loop_lags,
+        warm_tokenizer,
+    )
+
+    from litellm.caching.redis_semantic_cache import RedisSemanticCache
+
+    warm_tokenizer("sem-embed")
+    cache = RedisSemanticCache.__new__(RedisSemanticCache)
+    cache.embedding_model = "sem-embed"
+    cache.embedding_max_input_tokens = 5
+    cache.embedding_timeout = 5
+
+    router = MagicMock()
+    router.get_configured_token_limits.return_value = (8191, None)
+    router.aembedding = AsyncMock(return_value={"data": [{"embedding": [0.1, 0.2]}]})
+    _proxy_with_router(monkeypatch, router, "sem-embed")
+
+    embedding, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(text * 100))
+
+    assert embedding == [0.1, 0.2]
+    assert _token_count("sem-embed", router.aembedding.call_args.kwargs["input"]) == 5
+    assert_loop_stayed_free(took, lags)

tests/test_litellm/integrations/compression_interception/test_compression_interception_handler.py

@@ -523,3 +523,28 @@ async def test_pre_call_hook_no_compression_records_no_savings(monkeypatch):
     await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
 
     assert "compression_savings" not in litellm_metadata
+
+
+@pytest.mark.asyncio
+async def test_pre_call_hook_counts_tokens_off_the_event_loop():
+    from tests.large_text import text
+    from tests.test_litellm.litellm_core_utils.event_loop_lag import (
+        assert_loop_stayed_free,
+        timed_with_loop_lags,
+        warm_tokenizer,
+    )
+
+    model = "anthropic/claude-fable-5"
+    warm_tokenizer(model)
+    logger = CompressionInterceptionLogger(compression_trigger=10_000_000)
+    messages = [{"role": "user", "content": text * 100}]
+    kwargs = {"model": model, "messages": messages}
+
+    result, took, lags = await timed_with_loop_lags(
+        lambda: logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
+    )
+
+    assert result is not None
+    assert result["messages"] is messages
+    assert "tools" not in result
+    assert_loop_stayed_free(took, lags)

tests/test_litellm/integrations/test_langsmith_init.py

@@ -1,5 +1,6 @@
 import asyncio
 import os
+from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
@@ -7,6 +8,7 @@ import pytest
 
 import litellm
 from litellm.integrations.langsmith import LangsmithLogger
+from litellm.types.integrations.langsmith import LangsmithQueueObject
 
 
 @pytest.fixture
@@ -531,3 +533,44 @@ class TestLangsmithRootRunIdConsistency:
 
         assert data["trace_id"] == "trace-1"
         assert data["dotted_order"] == dotted
+
+
+@pytest.mark.asyncio
+async def test_events_appended_during_flush_are_not_dropped():
+    logger = LangsmithLogger(langsmith_api_key="test-key", langsmith_project="test-project")
+    try:
+        sent_batches: Final[list[list[dict[str, str]]]] = []
+        late_event: Final = LangsmithQueueObject(
+            credentials=logger.default_credentials, data={"id": "late"}
+        )
+
+        async def fake_post(
+            url: str, json: dict[str, list[dict[str, str]]], headers: dict[str, str]
+        ) -> MagicMock:
+            if not sent_batches:
+                logger.log_queue.append(late_event)
+            sent_batches.append(json["post"])
+            response = MagicMock()
+            response.status_code = 200
+            response.raise_for_status = MagicMock()
+            return response
+
+        logger.async_httpx_client = MagicMock(post=AsyncMock(side_effect=fake_post))
+        logger.log_queue = [
+            LangsmithQueueObject(credentials=logger.default_credentials, data={"id": "a"}),
+            LangsmithQueueObject(credentials=logger.default_credentials, data={"id": "b"}),
+        ]
+
+        await logger.flush_queue()
+
+        assert [e["id"] for e in sent_batches[0]] == ["a", "b"]
+        assert logger.log_queue == [late_event]
+
+        await logger.flush_queue()
+
+        assert [e["id"] for e in sent_batches[1]] == ["late"]
+        assert logger.log_queue == []
+    finally:
+        if logger._flush_task is not None:
+            logger._flush_task.cancel()
+            await asyncio.gather(logger._flush_task, return_exceptions=True)

tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py

@@ -1,18 +1,18 @@
 """
 Unit tests for Prometheus invalid API key request filtering.
 
-Tests functionality that prevents invalid API key requests (401 status codes)
-from being recorded in Prometheus metrics.
+Tests the 401 detection helpers, that LLM-level metrics skip invalid API key
+requests, and that the proxy-level failed request counter still records them.
 """
 
 from unittest.mock import Mock, patch
 
 import pytest
+from fastapi import HTTPException
 from prometheus_client import REGISTRY
 
-
 from litellm.integrations.prometheus import PrometheusLogger
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
 
 
 @pytest.fixture(scope="function")
@@ -129,28 +129,29 @@ class TestSkipMetricsValidation:
 
 
 class TestAsyncHooks:
-    """Test async hook methods skip metrics for invalid API keys."""
-
-    @pytest.fixture
-    def mock_user_api_key(self):
-        """Create a mock UserAPIKeyAuth object."""
-        user_key = Mock(spec=UserAPIKeyAuth)
-        user_key.api_key = "test-key"
-        user_key.end_user_id = None
-        user_key.user_id = None
-        user_key.user_email = None
-        user_key.key_alias = None
-        user_key.team_id = None
-        user_key.team_alias = None
-        user_key.request_route = "/test"
-        return user_key
+    """Test how async hook methods treat invalid API key requests."""
 
     @pytest.mark.asyncio
-    async def test_post_call_failure_hook_skips_401(
-        self, prometheus_logger, mock_user_api_key
+    @pytest.mark.parametrize(
+        "exception",
+        [
+            HTTPException(
+                status_code=401,
+                detail="LiteLLM Virtual Key expected. Received=nota****tall, expected to start with 'sk-'.",
+            ),
+            ProxyException(
+                message="Authentication Error, Invalid proxy server token passed.",
+                type=ProxyErrorTypes.token_not_found_in_db,
+                param="key",
+                code=401,
+            ),
+        ],
+    )
+    async def test_post_call_failure_hook_counts_401_without_key_hash(
+        self, prometheus_logger, exception
     ):
-        exception = ExceptionWithCode("401")
-        exception.__class__.__name__ = "ProxyException"
+        unauthenticated = UserAPIKeyAuth(request_route="/v1/chat/completions")
+        unauthenticated.api_key = "notakeyatall"
 
         with (
             patch.object(
@@ -160,15 +161,50 @@ class TestAsyncHooks:
                 prometheus_logger, "litellm_proxy_total_requests_metric"
             ) as mock_total,
         ):
-
             await prometheus_logger.async_post_call_failure_hook(
                 request_data={"model": "test-model"},
                 original_exception=exception,
-                user_api_key_dict=mock_user_api_key,
+                user_api_key_dict=unauthenticated,
             )
 
-            mock_failed.labels.assert_not_called()
-            mock_total.labels.assert_not_called()
+        failed_labels = mock_failed.labels.call_args.kwargs
+        assert failed_labels["exception_status"] == "401"
+        assert failed_labels["hashed_api_key"] is None
+        assert failed_labels["route"] == "/v1/chat/completions"
+        mock_failed.labels.return_value.inc.assert_called_once()
+        assert mock_total.labels.call_args.kwargs["status_code"] == "401"
+        mock_total.labels.return_value.inc.assert_called_once()
+
+    @pytest.mark.asyncio
+    async def test_post_call_failure_hook_keeps_resolved_identity_labels_for_401(
+        self, prometheus_logger
+    ):
+        expired_key = UserAPIKeyAuth(
+            api_key="sk-expired",
+            key_alias="expired-alias",
+            team_id="team-1",
+        )
+        exception = ProxyException(
+            message="Authentication Error - Expired Key.",
+            type=ProxyErrorTypes.expired_key,
+            param="key",
+            code=401,
+        )
+
+        with patch.object(
+            prometheus_logger, "litellm_proxy_failed_requests_metric"
+        ) as mock_failed:
+            await prometheus_logger.async_post_call_failure_hook(
+                request_data={"model": "test-model"},
+                original_exception=exception,
+                user_api_key_dict=expired_key,
+            )
+
+        failed_labels = mock_failed.labels.call_args.kwargs
+        assert failed_labels["exception_status"] == "401"
+        assert failed_labels["hashed_api_key"] is None
+        assert failed_labels["api_key_alias"] == "expired-alias"
+        assert failed_labels["team"] == "team-1"
 
     @pytest.mark.asyncio
     async def test_log_failure_event_skips_401(self, prometheus_logger):

tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py

@@ -2321,36 +2321,6 @@ def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities(_local_mo
         )
 
 
-@pytest.mark.parametrize(
-    "model,expected_mode,expected_input,expected_output,expected_cache_read",
-    [
-        ("azure/gpt-5.5", "chat", 5e-6, 3e-5, 5e-7),
-        ("azure/gpt-5.5-2026-04-23", "chat", 5e-6, 3e-5, 5e-7),
-        ("azure/gpt-5.5-pro", "responses", 3e-5, 1.8e-4, 3e-6),
-        ("azure/gpt-5.5-pro-2026-04-23", "responses", 3e-5, 1.8e-4, 3e-6),
-    ],
-)
-def test_azure_gpt55_entries_present_with_correct_pricing(_local_model_cost_map, 
-    model, expected_mode, expected_input, expected_output, expected_cache_read
-):
-    """Day-0 Azure entries for GPT-5.5 mirror the OpenAI pricing structure.
-
-    Pricing parity with openai/gpt-5.5* (verified against OpenAI's pricing page
-    on 2026-04-24): $5/$30 input/output per 1M for chat, $30/$180 for pro.
-    Cache discount is 10% of input.
-    """
-
-    m = litellm.model_cost[model]
-    assert m["litellm_provider"] == "azure"
-    assert m["mode"] == expected_mode
-    assert m["input_cost_per_token"] == expected_input
-    assert m["output_cost_per_token"] == expected_output
-    assert m["cache_read_input_token_cost"] == expected_cache_read
-    # Long-context window inherited from gpt-5.4 / openai gpt-5.5.
-    assert m["max_input_tokens"] == 1050000
-    assert m["max_output_tokens"] == 128000
-
-
 @pytest.mark.parametrize(
     "model,expected_none,expected_minimal,expected_xhigh",
     [
@@ -3414,8 +3384,6 @@ def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map):
 # ---------------------------------------------------------------------------
 
 
-
-
 @pytest.mark.parametrize("model", ["gpt-5.4", "gpt-realtime-2.1", "gpt-realtime-2.1-mini"])
 @pytest.mark.parametrize("data_residency", ["eu", "us"])
 def test_data_residency_applies_uplift(data_residency, model, _local_model_cost_map):
@@ -4556,20 +4524,6 @@ GEMINI_DAY0_LAUNCH_PRICING = [
 ]
 
 
-@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_DAY0_LAUNCH_PRICING)
-def test_gemini_36_flash_and_35_flash_lite_launch_pricing(_local_model_cost_map, model, input_cost, output_cost, cache_read_cost):
-
-    model_cost_map = litellm.model_cost[model]
-    assert model_cost_map["input_cost_per_token"] == input_cost
-    assert model_cost_map["output_cost_per_token"] == output_cost
-    assert model_cost_map["output_cost_per_reasoning_token"] == output_cost
-    assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost
-    assert model_cost_map["mode"] == "chat"
-    assert model_cost_map["supports_reasoning"] is True
-    assert model_cost_map["supports_function_calling"] is True
-    assert model_cost_map["max_input_tokens"] == 1048576
-
-
 def test_generic_cost_per_token_gemini_36_flash(_local_model_cost_map):
 
     usage = Usage(
@@ -4598,44 +4552,6 @@ GEMINI_36_FLASH_SERVICE_TIER_PRICING = [
 ]
 
 
-@pytest.mark.parametrize(
-    "service_tier,input_rate,output_rate,cache_read_rate", GEMINI_36_FLASH_SERVICE_TIER_PRICING
-)
-@pytest.mark.parametrize(
-    "model", ["gemini-3.6-flash", "gemini/gemini-3.6-flash", "vertex_ai/gemini-3.6-flash"]
-)
-def test_gemini_36_flash_service_tier_introductory_pricing(
-    model, service_tier, input_rate, output_rate, cache_read_rate, _local_model_cost_map
-):
-    """Regression: every 3.6 Flash tier is on Google's introductory rates through 2026-12-31,
-    so flex and priority requests must not be billed at the post-introductory rates."""
-    usage = Usage(
-        prompt_tokens=1_000,
-        completion_tokens=500,
-        total_tokens=1_500,
-        prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200, text_tokens=800),
-    )
-
-    prompt_cost, completion_cost = generic_cost_per_token(
-        model=model.split("/")[-1],
-        usage=usage,
-        custom_llm_provider=model.split("/")[0] if "/" in model else "gemini",
-        service_tier=service_tier,
-    )
-
-    assert prompt_cost == pytest.approx(800 * input_rate + 200 * cache_read_rate, rel=1e-9)
-    assert completion_cost == pytest.approx(500 * output_rate, rel=1e-9)
-
-
-@pytest.mark.parametrize(
-    "model", ["gemini-3.6-flash", "gemini/gemini-3.6-flash", "vertex_ai/gemini-3.6-flash"]
-)
-def test_gemini_36_flash_batch_introductory_pricing(model, _local_model_cost_map):
-    model_cost_map = litellm.model_cost[model]
-    assert model_cost_map["input_cost_per_token_batches"] == 3.75e-07
-    assert model_cost_map["output_cost_per_token_batches"] == 1.875e-06
-
-
 def test_generic_cost_per_token_gemini_35_flash_lite(_local_model_cost_map):
 
     usage = Usage(
@@ -4667,43 +4583,6 @@ GEMINI_35_FLASH_LITE_TIER_RATES_BY_SURFACE = [
 ]
 
 
-@pytest.mark.parametrize(
-    "custom_llm_provider,service_tier,input_rate,output_rate,cache_read_rate",
-    GEMINI_35_FLASH_LITE_TIER_RATES_BY_SURFACE,
-)
-def test_gemini_35_flash_lite_service_tier_pricing(
-    custom_llm_provider, service_tier, input_rate, output_rate, cache_read_rate, _local_model_cost_map
-):
-    """Regression: Vertex publishes flash-lite flex context caching at $0.015/M while the
-    Gemini API publishes $0.02/M, so vertex_ai flex cache reads must bill 1.5e-08/token
-    instead of the 2e-08 the map used to carry, without disturbing the Gemini API rate."""
-    usage = Usage(
-        prompt_tokens=1_000,
-        completion_tokens=500,
-        total_tokens=1_500,
-        prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200, text_tokens=800),
-    )
-
-    prompt_cost, completion_cost = generic_cost_per_token(
-        model="gemini-3.5-flash-lite",
-        usage=usage,
-        custom_llm_provider=custom_llm_provider,
-        service_tier=service_tier,
-    )
-
-    assert prompt_cost == pytest.approx(800 * input_rate + 200 * cache_read_rate, rel=1e-9)
-    assert completion_cost == pytest.approx(500 * output_rate, rel=1e-9)
-
-
-def test_gemini_35_flash_lite_flex_cache_read_map_entries(_local_model_cost_map):
-    """Each map entry carries its own surface's published flex cache-read rate: the bare
-    and vertex_ai keys are the Vertex surface at $0.015/M, the gemini key is the Gemini
-    API surface at $0.02/M."""
-    assert litellm.model_cost["gemini-3.5-flash-lite"]["cache_read_input_token_cost_flex"] == 1.5e-08
-    assert litellm.model_cost["vertex_ai/gemini-3.5-flash-lite"]["cache_read_input_token_cost_flex"] == 1.5e-08
-    assert litellm.model_cost["gemini/gemini-3.5-flash-lite"]["cache_read_input_token_cost_flex"] == 2e-08
-
-
 @pytest.mark.parametrize(
     "service_tier,input_rate,cache_read_rate,cache_write_rate,output_rate",
     [
@@ -4932,19 +4811,6 @@ GEMINI_37_FLASH_LAUNCH_PRICING = [
 ]
 
 
-@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_37_FLASH_LAUNCH_PRICING)
-def test_gemini_37_flash_launch_pricing(model, input_cost, output_cost, cache_read_cost, _local_model_cost_map):
-    model_cost_map = litellm.model_cost[model]
-    assert model_cost_map["input_cost_per_token"] == input_cost
-    assert model_cost_map["output_cost_per_token"] == output_cost
-    assert model_cost_map["output_cost_per_reasoning_token"] == output_cost
-    assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost
-    assert model_cost_map["mode"] == "chat"
-    assert model_cost_map["supports_reasoning"] is True
-    assert model_cost_map["supports_function_calling"] is True
-    assert model_cost_map["max_input_tokens"] == 1048576
-
-
 def test_generic_cost_per_token_gemini_37_flash(_local_model_cost_map):
     usage = Usage(
         prompt_tokens=1000,
@@ -4972,19 +4838,6 @@ GEMINI_38_FLASH_LAUNCH_PRICING = [
 ]
 
 
-@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_38_FLASH_LAUNCH_PRICING)
-def test_gemini_38_flash_launch_pricing(model, input_cost, output_cost, cache_read_cost, _local_model_cost_map):
-    model_cost_map = litellm.model_cost[model]
-    assert model_cost_map["input_cost_per_token"] == input_cost
-    assert model_cost_map["output_cost_per_token"] == output_cost
-    assert model_cost_map["output_cost_per_reasoning_token"] == output_cost
-    assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost
-    assert model_cost_map["mode"] == "chat"
-    assert model_cost_map["supports_reasoning"] is True
-    assert model_cost_map["supports_function_calling"] is True
-    assert model_cost_map["max_input_tokens"] == 1048576
-
-
 GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH = (
     "input_cost_per_token",
     "output_cost_per_token",
@@ -5045,20 +4898,6 @@ def test_generic_cost_per_token_gemini_38_flash(_local_model_cost_map):
     assert completion_cost == pytest.approx(0.001875)
 
 
-def test_grok_46_launch_pricing(_local_model_cost_map):
-    model_cost_map = litellm.model_cost["xai/grok-4.6"]
-    assert model_cost_map["input_cost_per_token"] == 2e-06
-    assert model_cost_map["output_cost_per_token"] == 6e-06
-    assert model_cost_map["cache_read_input_token_cost"] == 5e-07
-    assert model_cost_map["input_cost_per_token_above_200k_tokens"] == 4e-06
-    assert model_cost_map["output_cost_per_token_above_200k_tokens"] == 1.2e-05
-    assert model_cost_map["cache_read_input_token_cost_above_200k_tokens"] == 1e-06
-    assert model_cost_map["mode"] == "chat"
-    assert model_cost_map["supports_reasoning"] is True
-    assert model_cost_map["supports_function_calling"] is True
-    assert model_cost_map["max_input_tokens"] == 500000
-
-
 def test_generic_cost_per_token_grok_46(_local_model_cost_map):
     usage = Usage(
         prompt_tokens=1_000,

tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py

@@ -1,6 +1,4 @@
-import json
 from collections.abc import Mapping, Sequence
-from pathlib import Path
 
 import pytest
 
@@ -892,29 +890,6 @@ def test_gpt_4o_mini_snapshot_bills_web_search_like_its_alias(
     assert snapshot_cost == alias_cost == 0.025
 
 
-def test_gpt_4o_mini_web_search_price_matches_in_both_cost_maps():
-    repo_root = Path(__file__).parents[4]
-    cost_maps = tuple(
-        json.loads((repo_root / path).read_text(encoding="utf-8"))
-        for path in (
-            "model_prices_and_context_window.json",
-            "litellm/model_prices_and_context_window_backup.json",
-        )
-    )
-    canonical, backup = cost_maps
-    expected_search_price = {
-        "search_context_size_low": 0.025,
-        "search_context_size_medium": 0.025,
-        "search_context_size_high": 0.025,
-    }
-    for model_name in ("gpt-4o-mini", "gpt-4o-mini-2024-07-18"):
-        canonical_entry = canonical[model_name]
-        backup_entry = backup[model_name]
-        assert canonical_entry["search_context_cost_per_query"] == expected_search_price
-        assert backup_entry["search_context_cost_per_query"] == expected_search_price
-        assert canonical_entry == backup_entry
-
-
 # Note: File search integration test removed due to complex annotation detection logic
 # The unit tests in test_azure_assistant_cost_tracking.py provide comprehensive coverage
 

tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py

@@ -627,17 +627,6 @@ def test_shipped_adaptive_rule_requires_claude_prefix(shipped_cost_map):
         litellm.get_model_info(model)
 
 
-def test_shipped_exact_entry_beats_rules(shipped_cost_map):
-    model = "us.anthropic.claude-sonnet-4-6"
-    assert model in litellm.model_cost
-    info = litellm.get_model_info(model, custom_llm_provider="bedrock")
-    assert info["litellm_provider"] == "bedrock_converse"
-    assert info["input_cost_per_token"] == 3.3e-06
-    assert info["max_input_tokens"] == 1000000
-    assert info["supports_adaptive_thinking"] is True
-    assert info.get("supports_mid_conversation_system") is None
-
-
 def test_shipped_rules_lose_to_exact_entries_across_cost_ladder_variants(shipped_cost_map):
     """A route-mangled variant of an exactly-mapped model must never resolve from
     rules. The cost calculator tries model-name variants in order; a rule-derived

tests/test_litellm/litellm_core_utils/test_get_llm_provider_logic.py

@@ -0,0 +1,55 @@
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm import CustomLLM
+from litellm.litellm_core_utils.get_llm_provider_logic import (
+    get_llm_provider,
+    is_registered_custom_provider,
+)
+
+CUSTOM_PROVIDER: Final = "test-onprem-llm"
+
+
+@pytest.fixture
+def registered_custom_provider(monkeypatch: pytest.MonkeyPatch) -> str:
+    monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": CUSTOM_PROVIDER, "custom_handler": CustomLLM()}])
+    monkeypatch.setattr(litellm, "provider_list", list(litellm.provider_list))
+    monkeypatch.setattr(litellm, "_custom_providers", list(litellm._custom_providers))
+    return CUSTOM_PROVIDER
+
+
+def test_get_llm_provider_resolves_custom_provider_map_prefix_before_first_completion(
+    registered_custom_provider: str,
+) -> None:
+    assert registered_custom_provider not in litellm.provider_list
+
+    model, provider, dynamic_api_key, api_base = get_llm_provider(model=f"{registered_custom_provider}/my-model")
+
+    assert (model, provider, dynamic_api_key, api_base) == ("my-model", registered_custom_provider, None, None)
+
+
+def test_get_llm_provider_strips_prefix_when_custom_provider_passed_explicitly(
+    registered_custom_provider: str,
+) -> None:
+    model, provider, _, api_base = get_llm_provider(
+        model="my-model",
+        custom_llm_provider=registered_custom_provider,
+        api_base="http://onprem.internal:8080",
+    )
+
+    assert (model, provider, api_base) == ("my-model", registered_custom_provider, "http://onprem.internal:8080")
+
+
+def test_get_llm_provider_still_rejects_unregistered_prefix(registered_custom_provider: str) -> None:
+    with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"):
+        get_llm_provider(model="not-registered-llm/my-model")
+
+
+@pytest.mark.parametrize(
+    ("candidate", "expected"),
+    [(CUSTOM_PROVIDER, True), ("not-registered-llm", False), (None, False), ("", False)],
+)
+def test_is_registered_custom_provider(registered_custom_provider: str, candidate: str | None, expected: bool) -> None:
+    assert is_registered_custom_provider(candidate) is expected

tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py

@@ -225,36 +225,6 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
         assert "supports_adaptive_thinking" not in backup[non_adaptive], non_adaptive
 
 
-@pytest.mark.parametrize(
-    "cost_map",
-    [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
-    ids=["root", "bundled_backup"],
-)
-def test_azure_ai_claude_1m_context_entries(cost_map: dict):
-    """Microsoft Foundry serves a 1M-token context window for Opus 4.6+ and Sonnet
-    4.6+, so the ``azure_ai`` entries must not advertise the 200k cap that made
-    context-aware clients compact prompts early (LIT-4406). Both the root map (used
-    by default network loading) and the bundled fallback are checked so the two can
-    never drift apart."""
-    for model in [
-        "azure_ai/claude-opus-4-6",
-        "azure_ai/claude-opus-4-7",
-        "azure_ai/claude-opus-4-8",
-        "azure_ai/claude-opus-5",
-        "azure_ai/claude-sonnet-5",
-        "azure_ai/claude-sonnet-4-6",
-    ]:
-        assert cost_map[model]["max_input_tokens"] == 1000000, model
-
-    for model in [
-        "azure_ai/claude-opus-4-1",
-        "azure_ai/claude-opus-4-5",
-        "azure_ai/claude-sonnet-4-5",
-        "azure_ai/claude-haiku-4-5",
-    ]:
-        assert cost_map[model]["max_input_tokens"] == 200000, model
-
-
 # OpenRouter headline rates from GET https://openrouter.ai/api/v1/models.
 # These were the catalog values that disagreed with that API (and, for the
 # two spotlight models, the public model pages that their source fields cite).
@@ -278,34 +248,6 @@ _OPENROUTER_STALE_COSTS = {
 }
 
 
-@pytest.mark.parametrize(
-    "cost_map",
-    [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
-    ids=["root", "bundled_backup"],
-)
-def test_openrouter_catalog_costs_match_live_headline_rates(cost_map: dict):
-    """openrouter/* spend tracking reads these catalog fields. The values must
-    stay aligned with OpenRouter's published headline rate, not the stale
-    figures that over/under-counted by up to 30x. Both maps are checked so
-    the root file and bundled backup cannot drift apart."""
-    control = cost_map["openrouter/anthropic/claude-opus-5"]
-    assert control["input_cost_per_token"] == 5e-06
-    assert control["output_cost_per_token"] == 2.5e-05
-    assert control["cache_read_input_token_cost"] == 5e-07
-
-    for model, (inp, out, cache) in _OPENROUTER_LIVE_COSTS.items():
-        entry = cost_map[model]
-        assert entry["input_cost_per_token"] == inp, model
-        assert entry["output_cost_per_token"] == out, model
-        if cache is not None:
-            assert entry["cache_read_input_token_cost"] == cache, model
-
-    for model, (stale_in, stale_out) in _OPENROUTER_STALE_COSTS.items():
-        entry = cost_map[model]
-        assert entry["input_cost_per_token"] != stale_in, model
-        assert entry["output_cost_per_token"] != stale_out, model
-
-
 def test_get_model_cost_map_stamps_loaded_at():
     """The load time feeds each pod's reload-due decision; a load that does not stamp it
     would make manual reload requests race the proxy's startup"""

tests/test_litellm/litellm_core_utils/test_streaming_handler.py

@@ -4927,3 +4927,50 @@ class TestStableStreamingResponseId:
         )
         wrapper.response_id = "chatcmpl-from-provider"
         assert wrapper.model_response_creator().id == "chatcmpl-from-provider"
+
+
+@pytest.mark.asyncio
+async def test_async_stream_without_usage_counts_tokens_off_the_event_loop():
+    from tests.large_text import text
+    from tests.test_litellm.litellm_core_utils.event_loop_lag import (
+        assert_loop_stayed_free,
+        timed_with_loop_lags,
+        warm_tokenizer,
+    )
+
+    model = "gpt-5.6-luna"
+    warm_tokenizer(model)
+    messages = [{"role": "user", "content": text * 100}]
+    content_chunks = [_make_chunk(text) for _ in range(100)]
+    stop_chunk = ModelResponseStream(
+        id="test",
+        created=1741037890,
+        model=model,
+        choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")],
+    )
+    logging_obj = Logging(
+        model=model,
+        messages=messages,
+        stream=True,
+        call_type="acompletion",
+        start_time=time.time(),
+        litellm_call_id="12345",
+        function_id="1245",
+    )
+    wrapper = CustomStreamWrapper(
+        completion_stream=ModelResponseListIterator(model_responses=content_chunks + [stop_chunk]),
+        model=model,
+        custom_llm_provider="openai",
+        logging_obj=logging_obj,
+        stream_options={"include_usage": True},
+    )
+
+    async def consume() -> list[ModelResponseStream]:
+        return [chunk async for chunk in wrapper]
+
+    chunks, took, lags = await timed_with_loop_lags(consume)
+
+    assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == text * 100
+    assert chunks[-1].usage.prompt_tokens > 100_000
+    assert chunks[-1].usage.completion_tokens > 100_000
+    assert_loop_stayed_free(took, lags)

tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py

@@ -2605,3 +2605,34 @@ def test_build_summary_messages_keeps_midturn_system_correction_in_place():
     assert summary_messages[0]["content"] == "caller system prompt"
     assert summary_messages[2]["content"] == "use the corrected result"
     assert summary_messages[-1]["content"] == "summarize the conversation"
+
+
+async def test_threshold_check_counts_tokens_off_the_event_loop(monkeypatch):
+    from tests.large_text import text
+    from tests.test_litellm.litellm_core_utils.event_loop_lag import (
+        assert_loop_stayed_free,
+        timed_with_loop_lags,
+        warm_tokenizer,
+    )
+
+    from litellm.llms.anthropic.experimental_pass_through.context_management.constants import (
+        COMPACT_SUMMARY_MODEL_SETTING_KEY,
+    )
+    from litellm.proxy.proxy_server import general_settings
+
+    monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "claude-haiku-4-5")
+    warm_tokenizer(MODEL)
+    messages = [{"role": "user", "content": text * 100}, *_simple_messages()]
+    result, took, lags = await timed_with_loop_lags(
+        lambda: apply_compact_20260112(
+            model=MODEL,
+            messages=messages,
+            tools=None,
+            system=None,
+            edit_spec={"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 10_000_000}},
+        )
+    )
+
+    assert result.messages == messages
+    assert result.compaction_block is None
+    assert_loop_stayed_free(took, lags)

tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_dispatcher.py

@@ -129,3 +129,35 @@ async def test_malformed_edit_entries_are_skipped():
     )
     assert result.applied_edits == []
     assert result.messages == messages
+
+
+async def test_sync_editor_counts_tokens_off_the_event_loop():
+    from tests.large_text import text
+    from tests.test_litellm.litellm_core_utils.event_loop_lag import (
+        assert_loop_stayed_free,
+        timed_with_loop_lags,
+        warm_tokenizer,
+    )
+
+    warm_tokenizer(MODEL)
+    messages = [{"role": "user", "content": text * 100}, *_history_with_two_tool_pairs()]
+
+    result, took, lags = await timed_with_loop_lags(
+        lambda: apply_context_management(
+            model=MODEL,
+            messages=messages,
+            tools=None,
+            system=None,
+            context_management_spec={
+                "edits": [
+                    {
+                        "type": "clear_tool_uses_20250919",
+                        "trigger": {"type": "input_tokens", "value": 10_000_000},
+                    }
+                ]
+            },
+        )
+    )
+
+    assert result.messages == messages
+    assert_loop_stayed_free(took, lags)

tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py

@@ -12,7 +12,6 @@ REPO_ROOT: Final = Path(__file__).parents[4]
 MAIN_COST_MAP: Final = REPO_ROOT / "model_prices_and_context_window.json"
 BACKUP_COST_MAP: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
 COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
-AZURE_PRICING_PREFIX: Final = "https://azure.microsoft.com/en-us/pricing/details/"
 A_MILLION: Final = 1_000_000
 AN_HOUR_IN_SECONDS: Final = 3600
 
@@ -76,7 +75,9 @@ def test_azure_ai_catalog_name_prices_the_same_in_any_casing(catalog_name: str)
 @pytest.mark.usefixtures("local_model_cost_map")
 @pytest.mark.parametrize("catalog_name", GROK_4_20_NAMES)
 def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalog_name: str) -> None:
-    uncached_prompt_cost, _ = cost_per_token(model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0)
+    uncached_prompt_cost, _ = cost_per_token(
+        model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0
+    )
     cached_prompt_cost, _ = cost_per_token(
         model=f"azure_ai/{catalog_name}",
         prompt_tokens=A_MILLION,
@@ -100,7 +101,6 @@ def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> No
     main_entry = _cost_map_entry(MAIN_COST_MAP, catalog_name)
     backup_entry = _cost_map_entry(BACKUP_COST_MAP, catalog_name)
 
-    assert str(main_entry["source"]).startswith(AZURE_PRICING_PREFIX)
     assert backup_entry == main_entry
 
 

tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py

@@ -1,4 +1,3 @@
-import json
 from pathlib import Path
 
 import pytest
@@ -24,19 +23,6 @@ def _ocr_response(model: str, pages_processed: int) -> OCRResponse:
     )
 
 
-@pytest.mark.parametrize("cost_map_path", COST_MAPS, ids=lambda path: path.name)
-@pytest.mark.parametrize("model, provider", MODELS)
-def test_pricing_entry(cost_map_path: Path, model: str, provider: str) -> None:
-    with open(cost_map_path) as f:
-        info = json.load(f).get(model)
-
-    assert info is not None, f"{model} missing from {cost_map_path.name}"
-    assert info["litellm_provider"] == provider
-    assert info["mode"] == "ocr"
-    assert info["supported_endpoints"] == ["/v1/ocr"]
-    assert info["ocr_cost_per_page"] == COST_PER_PAGE
-
-
 @pytest.mark.parametrize("model, provider", MODELS)
 def test_model_info_resolves_ocr_mode_and_price(local_model_cost_map, model: str, provider: str) -> None:
     info = litellm.get_model_info(model=model, custom_llm_provider=provider)

tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py

@@ -163,23 +163,6 @@ def test_legacy_endpoint_names_still_resolve(local_model_cost_map: None) -> None
     assert completion_cost == pytest.approx(100 * info["output_cost_per_token"])
 
 
-@pytest.mark.parametrize("model", NEW_MODELS)
-def test_new_models_price_at_published_dbu_rates(local_model_cost_map: None, model: str) -> None:
-    info: Final = _model_info(model)
-
-    for field, dbu_per_million in zip(PRICE_FIELDS, PUBLISHED_DBU_PER_MILLION[model]):
-        assert info[field] == _dollars_per_token(dbu_per_million), field
-
-
-@pytest.mark.parametrize("model", sorted(set(PUBLISHED_DBU_PER_MILLION) - set(ENTRIES_STORING_PROMOTIONAL_RATE)))
-def test_cache_rates_derive_from_published_cache_dbu(local_model_cost_map: None, model: str) -> None:
-    info: Final = _model_info(model)
-    cache_dbu_per_million: Final = PUBLISHED_DBU_PER_MILLION[model][2:]
-
-    for field, dbu_per_million in zip(CACHE_FIELDS, cache_dbu_per_million):
-        assert info[field] == _dollars_per_token(dbu_per_million), field
-
-
 @pytest.mark.parametrize("model", NEW_MODELS)
 def test_new_models_carry_cache_pricing(local_model_cost_map: None, model: str) -> None:
     info: Final = _model_info(model)
@@ -255,38 +238,3 @@ def test_sonnet_5_ships_standard_rates_not_introductory(local_model_cost_map: No
 
     for field in PRICE_FIELDS:
         assert sonnet_5[field] == pytest.approx(sonnet_4_6[field]), field
-
-
-@pytest.mark.parametrize("model", ENTRIES_STORING_PROMOTIONAL_RATE)
-def test_entries_storing_the_promotional_rate_price_below_the_published_table(
-    local_model_cost_map: None,
-    model: str,
-) -> None:
-    info: Final = _model_info(model)
-    input_dbu, output_dbu, _, _ = PUBLISHED_DBU_PER_MILLION[model]
-    expiry_hint: Final = f"the gemini promotion expires {PROMOTION_EXPIRES}, after which the list rate applies"
-
-    assert info["input_cost_per_token"] == pytest.approx(
-        _dollars_per_token(input_dbu) * PROMOTIONAL_DISCOUNT, rel=2e-4
-    ), expiry_hint
-    assert info["output_cost_per_token"] == pytest.approx(
-        _dollars_per_token(output_dbu) * PROMOTIONAL_DISCOUNT, rel=2e-4
-    ), expiry_hint
-    assert info["cache_creation_input_token_cost"] == pytest.approx(info["input_cost_per_token"])
-    assert info["cache_read_input_token_cost"] == pytest.approx(0.1 * info["input_cost_per_token"])
-
-
-@pytest.mark.parametrize("model", ENTRIES_STORING_LIST_RATE_DESPITE_PROMOTION)
-def test_entries_storing_the_list_rate_bill_above_the_promotional_price(
-    local_model_cost_map: None,
-    model: str,
-) -> None:
-    info: Final = _model_info(model)
-    input_dbu, _, _, _ = PUBLISHED_DBU_PER_MILLION[model]
-    list_rate: Final = _dollars_per_token(input_dbu)
-
-    assert info["input_cost_per_token"] == pytest.approx(list_rate, rel=2e-4), (
-        f"{model} moved off the list rate; if it now stores the discount that runs to "
-        f"{PROMOTION_EXPIRES}, move it into ENTRIES_STORING_PROMOTIONAL_RATE"
-    )
-    assert info["cache_creation_input_token_cost"] == pytest.approx(info["input_cost_per_token"])

tests/test_litellm/llms/fireworks_ai/rerank/test_fireworks_ai_rerank_transformation.py

@@ -3,6 +3,7 @@ Tests for Fireworks AI rerank transformation functionality.
 """
 
 import json
+import uuid
 from unittest.mock import MagicMock
 
 import httpx
@@ -181,8 +182,7 @@ class TestFireworksAIRerankTransform:
         )
 
         # Verify response structure
-        # Fireworks AI doesn't return "id", so it uses "model" as the id
-        assert result.id == "accounts/fireworks/models/qwen3-reranker-8b"
+        assert uuid.UUID(result.id).version == 4
         assert len(result.results) == 2
         assert result.results[0]["index"] == 0
         assert result.results[0]["relevance_score"] == 0.95
@@ -229,16 +229,14 @@ class TestFireworksAIRerankTransform:
             logging_obj=mock_logging,
         )
 
-        # Fireworks AI doesn't return "id", so it uses "model" as the id
-        assert result.id == "accounts/fireworks/models/qwen3-reranker-8b"
+        assert uuid.UUID(result.id).version == 4
         assert len(result.results) == 2
         assert result.results[0]["index"] == 0
         assert result.results[0]["relevance_score"] == 0.95
         # Document should not be present
         assert "document" not in result.results[0]
 
-    def test_transform_rerank_response_missing_id(self):
-        """Test response transformation when id is missing (should use model name or generate UUID)."""
+    def test_transform_rerank_response_missing_id_stamps_a_fresh_id_per_call(self):
         response_data = {
             "object": "list",
             "model": "accounts/fireworks/models/qwen3-reranker-8b",
@@ -248,23 +246,22 @@ class TestFireworksAIRerankTransform:
             "usage": {"total_tokens": 10},
         }
 
-        mock_response = MagicMock(spec=httpx.Response)
-        mock_response.json.return_value = response_data
-        mock_response.status_code = 200
-        mock_response.headers = {}
+        def transform() -> str:
+            mock_response = MagicMock(spec=httpx.Response)
+            mock_response.json.return_value = response_data
+            mock_response.status_code = 200
+            mock_response.headers = {}
+            return self.config.transform_rerank_response(
+                model=self.model,
+                raw_response=mock_response,
+                model_response=RerankResponse(),
+                logging_obj=MagicMock(),
+            ).id
 
-        mock_logging = MagicMock()
-        model_response = RerankResponse()
+        first, second = transform(), transform()
 
-        result = self.config.transform_rerank_response(
-            model=self.model,
-            raw_response=mock_response,
-            model_response=model_response,
-            logging_obj=mock_logging,
-        )
-
-        # Should use model name when id is missing
-        assert result.id == "accounts/fireworks/models/qwen3-reranker-8b"
+        assert first != second
+        assert "accounts/fireworks/models/qwen3-reranker-8b" not in (first, second)
 
     def test_transform_rerank_response_missing_results(self):
         """Test that missing results raises ValueError."""

tests/test_litellm/llms/gemini/test_cost_calculator.py

@@ -452,33 +452,6 @@ def test_map_traffic_type_to_service_tier(
     )
 
 
-@pytest.mark.parametrize(
-    "model,custom_llm_provider,expected_cache_read_cost",
-    [
-        ("gemini/gemini-flash-latest", "gemini", 3e-08),
-        ("gemini/gemini-flash-lite-latest", "gemini", 1e-08),
-        ("gemini/gemini-2.5-flash-preview-09-2025", "gemini", 3e-08),
-        ("gemini/gemini-2.5-flash-lite-preview-06-17", "gemini", 1e-08),
-        ("vertex_ai/gemini-2.5-flash-preview-09-2025", "vertex_ai", 3e-08),
-        ("vertex_ai/gemini-2.5-flash-lite-preview-06-17", "vertex_ai", 1e-08),
-    ],
-)
-def test_flash_alias_cache_read_is_ten_percent_of_input(
-    monkeypatch, model, custom_llm_provider, expected_cache_read_cost
-):
-    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
-    monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
-
-    model_info = litellm.get_model_info(
-        model=model, custom_llm_provider=custom_llm_provider
-    )
-
-    assert model_info["cache_read_input_token_cost"] == expected_cache_read_cost
-    assert model_info["cache_read_input_token_cost"] == pytest.approx(
-        0.10 * model_info["input_cost_per_token"]
-    )
-
-
 @pytest.mark.parametrize(
     "prefixed,bare",
     [

tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py

@@ -5,7 +5,6 @@ for mistral-ocr-4-0 and mistral-ocr-latest, which now both resolve to
 OCR 4 at $4 / 1000 pages.
 """
 
-import json
 from pathlib import Path
 
 import pytest
@@ -45,12 +44,6 @@ def _annotated_ocr_response(model: str, pages_processed: int | None, annotation_
     )
 
 
-@pytest.mark.parametrize("model", ["mistral-ocr-4-0", "mistral-ocr-latest"])
-def test_model_info_ocr4_price(model: str) -> None:
-    info = litellm.get_model_info(model=f"mistral/{model}", custom_llm_provider="mistral")
-    assert info["ocr_cost_per_page"] == OCR4_COST_PER_PAGE
-
-
 @pytest.mark.parametrize("model", ["mistral-ocr-4-0", "mistral-ocr-latest"])
 @pytest.mark.parametrize("pages_processed", [1, 3, 10])
 def test_ocr4_cost_scales_with_pages(model: str, pages_processed: int) -> None:
@@ -63,20 +56,6 @@ def test_ocr4_cost_scales_with_pages(model: str, pages_processed: int) -> None:
     assert cost == pytest.approx(OCR4_COST_PER_PAGE * pages_processed)
 
 
-
-@pytest.mark.parametrize("cost_map_path", [MAIN_COST_MAP, BACKUP_COST_MAP])
-def test_ocr3_pricing_entry(cost_map_path: Path) -> None:
-    with open(cost_map_path) as f:
-        info = json.load(f).get(OCR3_MODEL)
-
-    assert info is not None, f"{OCR3_MODEL} missing from {cost_map_path.name}"
-    assert info["litellm_provider"] == "mistral"
-    assert info["mode"] == "ocr"
-    assert info["supported_endpoints"] == ["/v1/ocr"]
-    assert info["ocr_cost_per_page"] == OCR3_COST_PER_PAGE
-    assert info["annotation_cost_per_page"] == OCR3_ANNOTATION_COST_PER_PAGE
-
-
 def test_ocr3_model_info_price(local_model_cost_map) -> None:
     info = litellm.get_model_info(model=OCR3_MODEL, custom_llm_provider="mistral")
     assert info["ocr_cost_per_page"] == OCR3_COST_PER_PAGE

tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_integration.py

@@ -104,10 +104,12 @@ class TestVertexAIRerankIntegration:
             raw_response=mock_response,
             model_response=model_response,
             logging_obj=mock_logging,
+            request_data=request_data,
         )
 
         # Verify response structure
-        assert result.id == f"vertex_ai_rerank_{self.model}"
+        assert result.id.startswith("vertex_ai_rerank_")
+        assert result.id != f"vertex_ai_rerank_{self.model}"
         assert len(result.results) == 2
 
         # Results should be sorted by relevance score (descending)
@@ -116,8 +118,8 @@ class TestVertexAIRerankIntegration:
         assert result.results[1]["index"] == 0  # Second highest score
         assert result.results[1]["relevance_score"] == 0.92
 
-        # Verify metadata
-        assert result.meta["billed_units"]["search_units"] == 2
+        # Verify metadata: 4 input records bill as 1 search unit (ceil(4/100))
+        assert result.meta["billed_units"]["search_units"] == 1
 
     def test_return_documents_false_flow(self):
         """Test rerank flow when return_documents=False (ID-only response)."""

tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py

@@ -287,10 +287,11 @@ class TestVertexAIRerankTransform:
             raw_response=mock_response,
             model_response=model_response,
             logging_obj=mock_logging,
+            request_data={"records": [{"id": "0"}, {"id": "1"}]},
         )
 
         # Verify response structure
-        assert result.id == f"vertex_ai_rerank_{self.model}"
+        assert result.id.startswith("vertex_ai_rerank_")
         assert len(result.results) == 2
         assert result.results[0]["index"] == 1  # Converted back to 0-based index
         assert result.results[0]["relevance_score"] == 0.98
@@ -298,7 +299,7 @@ class TestVertexAIRerankTransform:
         assert result.results[1]["relevance_score"] == 0.64
 
         # Verify metadata
-        assert result.meta["billed_units"]["search_units"] == 2
+        assert result.meta["billed_units"]["search_units"] == 1
 
     def test_transform_rerank_response_with_ignore_record_details(self):
         """Test response transformation when ignoreRecordDetailsInResponse=true."""
@@ -326,6 +327,96 @@ class TestVertexAIRerankTransform:
         assert result.results[1]["index"] == 0
         assert result.results[1]["relevance_score"] == 1.0
 
+    def _build_response(self, num_records):
+        response_data = {
+            "records": [
+                {"id": str(i), "score": 1.0 - i / 1000, "title": "t", "content": "c"}
+                for i in range(num_records)
+            ]
+        }
+        mock_response = MagicMock(spec=httpx.Response)
+        mock_response.json.return_value = response_data
+        mock_response.text = json.dumps(response_data)
+        return mock_response
+
+    def test_search_units_from_input_records_not_truncated_response(self):
+        """
+        Regression for LIT-4995 part 1: search_units must be derived from the
+        billable input records (ceil(input / 100)), not from the response, which
+        Google truncates to topN.
+        """
+        documents = [f"doc {i}" for i in range(5)]
+        request_data = self.config.transform_rerank_request(
+            model=self.model,
+            optional_rerank_params={"query": "q", "documents": documents, "top_n": 2},
+            headers={},
+        )
+        # Google truncates the response to top_n=2 records
+        mock_response = self._build_response(num_records=2)
+
+        result = self.config.transform_rerank_response(
+            model=self.model,
+            raw_response=mock_response,
+            model_response=RerankResponse(),
+            logging_obj=MagicMock(),
+            request_data=request_data,
+        )
+
+        assert result.meta["billed_units"]["search_units"] == 1
+
+    def test_search_units_rounds_up_per_hundred_input_records(self):
+        """
+        Regression for LIT-4995 part 1: one query bills up to 100 input records,
+        so 150 input records is 2 search units regardless of the response size.
+        """
+        documents = [f"doc {i}" for i in range(150)]
+        request_data = self.config.transform_rerank_request(
+            model=self.model,
+            optional_rerank_params={"query": "q", "documents": documents, "top_n": 3},
+            headers={},
+        )
+        mock_response = self._build_response(num_records=3)
+
+        result = self.config.transform_rerank_response(
+            model=self.model,
+            raw_response=mock_response,
+            model_response=RerankResponse(),
+            logging_obj=MagicMock(),
+            request_data=request_data,
+        )
+
+        assert result.meta["billed_units"]["search_units"] == 2
+
+    def test_response_id_is_unique_per_request(self):
+        """
+        Regression for LIT-4995 part 2: response IDs must be unique per request,
+        not a constant derived only from the model name.
+        """
+        request_data = self.config.transform_rerank_request(
+            model=self.model,
+            optional_rerank_params={"query": "q", "documents": ["a", "b"]},
+            headers={},
+        )
+        mock_response = self._build_response(num_records=2)
+
+        first = self.config.transform_rerank_response(
+            model=self.model,
+            raw_response=mock_response,
+            model_response=RerankResponse(),
+            logging_obj=MagicMock(),
+            request_data=request_data,
+        )
+        second = self.config.transform_rerank_response(
+            model=self.model,
+            raw_response=mock_response,
+            model_response=RerankResponse(),
+            logging_obj=MagicMock(),
+            request_data=request_data,
+        )
+
+        assert first.id != second.id
+        assert first.id != f"vertex_ai_rerank_{self.model}"
+
     def test_transform_rerank_response_json_error(self):
         """Test response transformation with JSON parsing error."""
         mock_response = MagicMock(spec=httpx.Response)

tests/test_litellm/llms/voyage/rerank/test_voyage_rerank_transformation.py

@@ -3,6 +3,7 @@ Tests for Voyage AI rerank transformation functionality.
 """
 
 import json
+import uuid
 from unittest.mock import MagicMock, patch
 
 import httpx
@@ -258,6 +259,33 @@ class TestVoyageRerankTransform:
 
         assert "Failed to parse response" in str(exc_info.value)
 
+    def test_transform_rerank_response_without_id_stamps_a_fresh_id_per_call(self):
+        response_data = {
+            "object": "list",
+            "data": [{"relevance_score": 0.5, "index": 0}],
+            "model": "rerank-2.5",
+            "usage": {"total_tokens": 10},
+        }
+
+        def transform() -> str:
+            mock_response = MagicMock(spec=httpx.Response)
+            mock_response.json.return_value = response_data
+            mock_response.status_code = 200
+            mock_response.text = json.dumps(response_data)
+            mock_response.headers = {}
+            return self.config.transform_rerank_response(
+                model=self.model,
+                raw_response=mock_response,
+                model_response=RerankResponse(),
+                logging_obj=MagicMock(),
+            ).id
+
+        first, second = transform(), transform()
+
+        assert uuid.UUID(first).version == 4
+        assert first != second
+        assert f"voyage-rerank-{self.model}" not in (first, second)
+
     def test_get_supported_cohere_rerank_params(self):
         """Test getting supported parameters for Voyage AI rerank."""
         supported_params = self.config.get_supported_cohere_rerank_params(self.model)

tests/test_litellm/llms/watsonx/rerank/test_watsonx_rerank.py

@@ -120,9 +120,7 @@ class TestIBMWatsonXRerankTransform:
             logging_obj=mock_logging,
         )
 
-        # Verify response structure
-        # IBM watsonx.ai doesn't return "id", so it uses "model" as the id
-        assert result.id == "watsonx/cross-encoder/ms-marco-minilm-l-12-v2"
+        assert uuid.UUID(result.id).version == 4
         assert len(result.results) == 2
         assert result.results[0]["index"] == 0
         assert result.results[0]["relevance_score"] == 6.53515625
@@ -172,9 +170,7 @@ class TestIBMWatsonXRerankTransform:
             logging_obj=mock_logging,
         )
 
-        # Verify response structure
-        # IBM watsonx.ai doesn't return "id", so it uses "model" as the id
-        assert result.id == "watsonx/cross-encoder/ms-marco-minilm-l-12-v2"
+        assert uuid.UUID(result.id).version == 4
         assert len(result.results) == 2
 
         assert result.results[0]["index"] == 0
@@ -231,6 +227,30 @@ class TestIBMWatsonXRerankTransform:
                 logging_obj=mock_logging,
             )
 
+    def test_transform_rerank_response_without_id_stamps_a_fresh_id_per_call(self):
+        response_data = {
+            "model_id": self.model,
+            "results": [{"index": 0, "score": 1.5}],
+            "input_token_count": 12,
+        }
+
+        def transform() -> str:
+            mock_response = MagicMock(spec=httpx.Response)
+            mock_response.json.return_value = response_data
+            mock_response.status_code = 200
+            mock_response.headers = {}
+            return self.config.transform_rerank_response(
+                model=self.model,
+                raw_response=mock_response,
+                model_response=RerankResponse(),
+                logging_obj=MagicMock(),
+            ).id
+
+        first, second = transform(), transform()
+
+        assert first != second
+        assert self.model not in (first, second)
+
     def test_get_supported_cohere_rerank_params(self):
         """Test getting supported parameters for IBM watsonx.ai rerank."""
         supported_params = self.config.get_supported_cohere_rerank_params(self.model)

tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py

@@ -119,6 +119,67 @@ class TestXAIResponsesAPITransformation:
         assert tool["filters"]["allowed_domains"] == ["wikipedia.org", "x.ai"]
         assert tool["enable_image_understanding"] is True
 
+    def test_web_search_nested_filters_preserved(self):
+        """The documented nested 'filters' shape must reach xAI instead of being dropped"""
+        config = XAIResponsesAPIConfig()
+
+        params = ResponsesAPIOptionalRequestParams(
+            tools=[
+                {
+                    "type": "web_search",
+                    "filters": {"allowed_domains": ["grokipedia.com"], "excluded_domains": ["example.com"]},
+                }
+            ]
+        )
+
+        result = config.map_openai_params(
+            response_api_optional_params=params,
+            model="grok-4-1-fast",
+            drop_params=False,
+        )
+
+        tool = result["tools"][0]
+        assert tool["filters"]["allowed_domains"] == ["grokipedia.com"]
+        assert tool["filters"]["excluded_domains"] == ["example.com"]
+
+    def test_web_search_nested_filters_win_over_flat(self):
+        """Nested filters take precedence when both shapes are sent"""
+        config = XAIResponsesAPIConfig()
+
+        params = ResponsesAPIOptionalRequestParams(
+            tools=[
+                {
+                    "type": "web_search",
+                    "allowed_domains": ["flat.com"],
+                    "filters": {"allowed_domains": ["nested.com"]},
+                }
+            ]
+        )
+
+        result = config.map_openai_params(
+            response_api_optional_params=params,
+            model="grok-4-1-fast",
+            drop_params=False,
+        )
+
+        assert result["tools"][0]["filters"] == {"allowed_domains": ["nested.com"]}
+
+    def test_web_search_empty_nested_filters_win_over_flat(self):
+        """An explicit empty 'filters' object means unrestricted search, even when stale flat fields are present"""
+        config = XAIResponsesAPIConfig()
+
+        params = ResponsesAPIOptionalRequestParams(
+            tools=[{"type": "web_search", "allowed_domains": ["flat.com"], "filters": {}}]
+        )
+
+        result = config.map_openai_params(
+            response_api_optional_params=params,
+            model="grok-4-1-fast",
+            drop_params=False,
+        )
+
+        assert result["tools"][0] == {"type": "web_search"}
+
     def test_web_search_search_context_size_removed(self):
         """Test that search_context_size is removed from web_search tools"""
         config = XAIResponsesAPIConfig()

tests/test_litellm/llms/xai/test_xai_chat_transformation.py

@@ -124,6 +124,24 @@ class TestXAIParallelToolCalls:
         assert result["messages"][0]["role"] == "user"
 
 
+class TestXAIChatWebSearchOptions:
+    """XAI answers /chat/completions requests carrying web_search_options with a 410 (Live Search retired)"""
+
+    def test_transform_request_drops_web_search_options(self):
+        config = XAIChatConfig()
+
+        result = config.transform_request(
+            model="xai/grok-4.6",
+            messages=[{"role": "user", "content": "newest litellm version?"}],
+            optional_params={"web_search_options": {"search_context_size": "medium"}, "temperature": 0.5},
+            litellm_params={},
+            headers={},
+        )
+
+        assert "web_search_options" not in result
+        assert result["temperature"] == 0.5
+
+
 class TestXAIUsageNormalization:
     def test_preserves_reasoning_tokens_in_total_usage(self):
         usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=200)

tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py

@@ -54,9 +54,6 @@ CODE_SLUGS = (
     "xai/grok-code-fast-1",
     "xai/grok-code-fast-1-0825",
 )
-RETIREMENT_DATE = "2026-05-15"
-GROK_3_MINI_RETIREMENT_DATE = "2026-02-28"
-
 BASE_COST_FIELDS = ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")
 TIER_COST_FIELDS = (
     "input_cost_per_token_above_200k_tokens",
@@ -65,10 +62,6 @@ TIER_COST_FIELDS = (
 )
 
 
-def expected_retirement_date(slug: str) -> str:
-    return GROK_3_MINI_RETIREMENT_DATE if slug in GROK_3_MINI_SLUGS else RETIREMENT_DATE
-
-
 @pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS])
 def cost_map(request: pytest.FixtureRequest) -> dict:
     path = next(p for p in MAP_PATHS if p.name == request.param)
@@ -92,15 +85,9 @@ def test_code_slug_bills_at_grok_build_rate(cost_map: dict, slug: str):
         assert entry[field] == target[field], field
 
 
-@pytest.mark.parametrize("slug", (*REDIRECTED_SLUGS, *CODE_SLUGS))
-def test_redirected_slug_keeps_its_retirement_date(cost_map: dict, slug: str):
-    assert cost_map[slug]["deprecation_date"] == expected_retirement_date(slug)
-
-
 def test_a_live_xai_model_is_untouched(cost_map: dict):
     """Guard against the repricing leaking onto models xAI still serves directly."""
     assert cost_map["xai/grok-4.6"]["input_cost_per_token"] != cost_map[REDIRECT_TARGET]["input_cost_per_token"]
-    assert "deprecation_date" not in cost_map["xai/grok-4.6"]
 
 
 @pytest.mark.parametrize("slug", REDIRECTED_SLUGS)

tests/test_litellm/proxy/auth/test_auth_exception_handler.py

@@ -487,6 +487,34 @@ async def test_route_passed_to_post_call_failure_hook():
             assert call_args["user_api_key_dict"].request_route == test_route
 
 
+@pytest.mark.asyncio
+async def test_dynamic_route_normalized_on_auth_failure():
+    handler = UserAPIKeyAuthExceptionHandler()
+
+    with (
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook",
+            new_callable=AsyncMock,
+        ) as mock_post_call_failure_hook,
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.proxy_server.general_settings", {}
+        ),
+        pytest.raises(ProxyException),
+    ):
+        await handler._handle_authentication_error(
+            HTTPException(status_code=401, detail="Authentication Error, Invalid proxy server token passed"),
+            MagicMock(),
+            {},
+            "/v1/responses/resp_attacker_controlled_id",
+            None,
+            "sk-doesnotexist",
+        )
+
+    hook_kwargs = mock_post_call_failure_hook.call_args.kwargs
+    assert hook_kwargs["route"] == "/v1/responses/resp_attacker_controlled_id"
+    assert hook_kwargs["user_api_key_dict"].request_route == "/v1/responses/{response_id}"
+
+
 @pytest.mark.asyncio
 async def test_resolved_identity_exported_on_auth_failure():
     """Regression: when auth fails AFTER the key/team/user identity is resolved
@@ -795,6 +823,89 @@ async def test_auth_failure_ip_stamp_does_not_mutate_callers_request_data():
     assert request_data == {"model": "gpt-4o"}
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "request_data, metadata_key, route",
+    [
+        pytest.param({"model": "gpt-4o"}, "metadata", "/v1/chat/completions", id="chat_metadata"),
+        pytest.param({"litellm_metadata": {}}, "litellm_metadata", "/v1/responses", id="responses_litellm_metadata"),
+    ],
+)
+async def test_auth_failure_logs_user_agent(request_data: dict[str, object], metadata_key: str, route: str) -> None:
+    """Auth gate rejections never reach `add_litellm_data_to_request`, which is what
+    stamps `user_agent`, so the failure spend log and prometheus `user_agent` label
+    had nothing to identify an abusive client by."""
+    with (
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.auth.auth_exception_handler.seed_request_identity"
+        ),
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook",
+            new_callable=AsyncMock,
+            return_value=None,
+        ) as mock_hook,
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.proxy_server.general_settings",
+            {"allow_requests_on_db_unavailable": False},
+        ),
+    ):
+        with pytest.raises(ProxyException):
+            await UserAPIKeyAuthExceptionHandler._handle_authentication_error(
+                ProxyException(
+                    message="Invalid API key",
+                    type=ProxyErrorTypes.auth_error,
+                    param=None,
+                    code=status.HTTP_401_UNAUTHORIZED,
+                ),
+                _http_request(headers={"user-agent": "abusive-client/9.9"}),
+                request_data,
+                route,
+                None,
+                "sk-bad-key",
+            )
+
+    logged_metadata = mock_hook.call_args[1]["request_data"][metadata_key]
+    assert logged_metadata["user_agent"] == "abusive-client/9.9"
+    assert logged_metadata["requester_ip_address"] == "10.1.2.3"
+
+
+@pytest.mark.asyncio
+async def test_auth_failure_without_headers_scope_still_raises_original_error() -> None:
+    """A request scope with no `headers` entry must surface the auth error itself, not a
+    `KeyError` from reading the User-Agent."""
+    with (
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.auth.auth_exception_handler.seed_request_identity"
+        ),
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook",
+            new_callable=AsyncMock,
+            return_value=None,
+        ) as mock_hook,
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.proxy_server.general_settings",
+            {"allow_requests_on_db_unavailable": False},
+        ),
+    ):
+        with pytest.raises(ProxyException) as exc_info:
+            await UserAPIKeyAuthExceptionHandler._handle_authentication_error(
+                ProxyException(
+                    message="Invalid API key",
+                    type=ProxyErrorTypes.auth_error,
+                    param=None,
+                    code=status.HTTP_401_UNAUTHORIZED,
+                ),
+                Request(scope={"type": "http"}),
+                {"model": "gpt-4o"},
+                "/v1/chat/completions",
+                None,
+                "sk-bad-key",
+            )
+
+    assert str(exc_info.value.code) == str(status.HTTP_401_UNAUTHORIZED)
+    assert "user_agent" not in mock_hook.call_args[1]["request_data"].get("metadata", {})
+
+
 def _marked_malformed_key_error() -> HTTPException:
     """Build the malformed-key 401 as its raise site does: marker stamped on it."""
     error = HTTPException(status_code=401, detail="LiteLLM Virtual Key expected. Received=test")