Merge branch 'main' into litellm_scope_cooldown_reads_to_candidates

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -4,6 +4,15 @@
      Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
      "## Affected release" with nothing under it must not appear in the final description -->
 
+## Description
+
+<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
+     Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
+     No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
+
+<details>
+<summary>Details</summary>
+
 ## TLDR
 
 <!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
@@ -164,3 +173,5 @@ Example checklists:
 ## Final Attestation
 
 - [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR
+
+</details>

litellm/caching/dual_cache.py

@@ -59,16 +59,6 @@ class PendingBatchRead:
     previous_access_times: dict[str, float | None]
 
 
-@dataclass(frozen=True, slots=True)
-class DeclaredBatchRead:
-    """A ``async_batch_get_cache`` split in two: the memory half done, the Redis half declared on a ``RedisBatch``
-    so it rides that batch's next round trip, resolved later with ``async_resolve_batch_get``."""
-
-    keys: tuple[str, ...]
-    pending: PendingBatchRead
-    result: BatchResult[Mapping[str, object]] | None
-
-
 def _log_deferred_increment_failure(future: asyncio.Future[float]) -> None:
     if future.cancelled():
         return
@@ -382,22 +372,6 @@ class DualCache(BaseCache):
                     await self.in_memory_cache.async_set_cache(key, value, **self._backfill_kwargs(kwargs))
         return merged
 
-    async def declare_batch_get(self, keys: Sequence[str], batch: RedisBatch) -> DeclaredBatchRead:
-        pending: Final = await self._prepare_batch_get(
-            list(keys),
-            local_only=False,
-            throttle_redis=False,
-        )
-        return DeclaredBatchRead(
-            keys=tuple(keys),
-            pending=pending,
-            result=batch.mget(pending.redis_keys) if pending.redis_keys else None,
-        )
-
-    async def async_resolve_batch_get(self, declared: DeclaredBatchRead) -> list[object | None]:
-        redis_result: Final = None if declared.result is None else await declared.result
-        return await self._apply_batch_get(declared.pending, redis_result)
-
     async def async_batch_get_cache(
         self,
         keys: list,

litellm/caching/redis_cache.py

@@ -134,7 +134,12 @@ _INCREMENT_WITH_FLOOR_LUA: Final = (
     "return count"
 )
 
+_INCREMENT_IF_PRESENT_LUA: Final = (
+    "if redis.call('EXISTS', KEYS[1]) == 0 then return false end return redis.call('INCRBYFLOAT', KEYS[1], ARGV[1])"
+)
+
 _LUA_COUNT: Final = TypeAdapter(int)
+_LUA_OPTIONAL_TOTAL: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
 _OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...])
 
 
@@ -675,7 +680,7 @@ class RedisCache(BaseCache):
         # redis namespaces
         self.namespace = namespace
         # for high traffic, we store the redis results in memory and then batch write to redis
-        self.redis_batch_writing_buffer: list = []
+        self.redis_batch_writing_buffer: list[tuple[str, object]] = []
         if redis_flush_size is None:
             self.redis_flush_size: int = 100
         else:
@@ -1568,6 +1573,46 @@ class RedisCache(BaseCache):
         count: Final = await _redis_client.eval(_INCREMENT_WITH_FLOOR_LUA, 1, namespaced_key, value, ttl)
         return _LUA_COUNT.validate_python(count)
 
+    @_redis_circuit_breaker_guard
+    async def async_increment_if_present(
+        self, key: str, value: float, parent_otel_span: Span | None = None
+    ) -> float | None:
+        _redis_client: Final = self._async_commands()
+        namespaced_key: Final = self.check_and_fix_namespace(key=key)
+        start_time: Final = time.time()
+        try:
+            total: Final = _LUA_OPTIONAL_TOTAL.validate_python(
+                await _redis_client.eval(_INCREMENT_IF_PRESENT_LUA, 1, namespaced_key, value)
+            )
+        except Exception as e:
+            failed_at: Final = time.time()
+            asyncio.create_task(
+                self.service_logger_obj.async_service_failure_hook(
+                    service=ServiceTypes.REDIS,
+                    duration=failed_at - start_time,
+                    error=e,
+                    call_type="async_increment_if_present",
+                    caller=_get_call_stack_info(),
+                    start_time=start_time,
+                    end_time=failed_at,
+                    parent_otel_span=parent_otel_span,
+                )
+            )
+            raise
+        end_time: Final = time.time()
+        asyncio.create_task(
+            self.service_logger_obj.async_service_success_hook(
+                service=ServiceTypes.REDIS,
+                duration=end_time - start_time,
+                call_type="async_increment_if_present",
+                caller=_get_call_stack_info(),
+                start_time=start_time,
+                end_time=end_time,
+                parent_otel_span=parent_otel_span,
+            )
+        )
+        return total
+
     async def flush_cache_buffer(self):
         print_verbose(f"flushing to redis....reached size of buffer {len(self.redis_batch_writing_buffer)}")
         await self.async_set_cache_pipeline(self.redis_batch_writing_buffer)

litellm/integrations/otel/README.md

@@ -91,7 +91,7 @@ way (`pod_lock`, `budget_reset`, `spend_queue`, `health_check`, `scheduler_queue
 rather than a bare `redis.set`; `tests/unit/test_internal_context.py` scans every
 module under `litellm/` and `enterprise/` that calls a shared cache or declares a
 read or write on the request batch (`reserve_redis_batch_reads`,
-`declare_batch_get`, `batch.mget`, `batch.set`, `batch.script`) and fails when
+`batch.mget`, `batch.set`, `batch.script`) and fails when
 one has no declared family, with the process-local `InMemoryCache` callers listed
 as the only exemptions. A batch op carries the family that was active when it was
 declared, so the routing prefetch armed before deployment selection

litellm/integrations/otel/model/spans.py

@@ -214,6 +214,7 @@ _SERVICE_VERB_BY_CALL_TYPE: Final[dict[str, str]] = {
     "increment_cache": "incr",
     "async_increment": "incr",
     "async_increment_pipeline": "incr",
+    "async_increment_if_present": "incr",
     "delete_cache": "delete",
     "async_delete_cache": "delete",
     "async_rpush": "rpush",

litellm/llms/bedrock/base_aws_llm.py

@@ -80,6 +80,12 @@ class _AssumeRoleParams(TypedDict):
     Tags: ReadOnly[NotRequired[tuple[AwsSessionTag, ...]]]
 
 
+class _StsClientKwargs(TypedDict):
+    verify: ReadOnly[bool | str]
+    endpoint_url: ReadOnly[NotRequired[str]]
+    region_name: ReadOnly[NotRequired[str]]
+
+
 def _assume_role_params(
     aws_role_name: str,
     aws_session_name: str,
@@ -275,7 +281,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
             "aws_session_tags",
         ]
 
-    def _get_ssl_verify(self, ssl_verify: bool | str | None = None):
+    def _get_ssl_verify(self, ssl_verify: bool | str | None = None) -> bool | str:
         """
         Get SSL verification setting for boto3 clients.
 
@@ -875,15 +881,18 @@ class BaseAWSLLM(SignsRequestsWithAWS):
         aws_sts_endpoint: str | None = None,
         ssl_verify: bool | str | None = None,
         aws_region_name: str | None = None,
-    ) -> dict:
+    ) -> _StsClientKwargs:
         """STS client kwargs with aligned endpoint_url and region_name (SigV4)."""
-        kwargs: Final[dict] = {"verify": self._get_ssl_verify(ssl_verify)}
-        if aws_sts_endpoint is not None:
-            kwargs["endpoint_url"] = aws_sts_endpoint
-        sts_region: Final = self._resolve_sts_region(aws_sts_endpoint, aws_region_name)
-        if sts_region is not None:
-            kwargs["region_name"] = sts_region
-        return kwargs
+        verify: Final = self._get_ssl_verify(ssl_verify)
+        match (aws_sts_endpoint, self._resolve_sts_region(aws_sts_endpoint, aws_region_name)):
+            case (None, None):
+                return _StsClientKwargs(verify=verify)
+            case (None, region):
+                return _StsClientKwargs(verify=verify, region_name=region)
+            case (endpoint, None):
+                return _StsClientKwargs(verify=verify, endpoint_url=endpoint)
+            case (endpoint, region):
+                return _StsClientKwargs(verify=verify, endpoint_url=endpoint, region_name=region)
 
     def get_aws_region_name_for_non_llm_api_calls(
         self,
@@ -1813,13 +1822,13 @@ async def run_aws_signing(
     )
 
 
-async def sign_request_off_loop_if_aws(
+async def run_off_loop_if_aws(
     provider_config: object,
-    sign_request: Callable[_SignParams, _SignedRequest],
+    call: Callable[_SignParams, _SignedRequest],
     /,
     *args: _SignParams.args,
-    **kwargs: _SignParams.kwargs,  # kwargs-ok: ParamSpec forwarding keeps the wrapped sign_request signature
+    **kwargs: _SignParams.kwargs,  # kwargs-ok: ParamSpec forwarding keeps the wrapped call's signature
 ) -> _SignedRequest:
     if isinstance(provider_config, SignsRequestsWithAWS):
-        return await run_aws_signing(sign_request, *args, **kwargs)
-    return sign_request(*args, **kwargs)
+        return await run_aws_signing(call, *args, **kwargs)
+    return call(*args, **kwargs)

litellm/llms/bedrock/claude_platform/messages_transformation.py

@@ -1,10 +1,11 @@
-from typing import Any, Final
+from typing import Final
 
 import litellm
 from litellm.llms.anthropic.pass_through.messages.transformation import (
     DEFAULT_ANTHROPIC_API_VERSION,
     AnthropicMessagesConfig,
 )
+from litellm.llms.bedrock.base_aws_llm import run_aws_signing
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.router import GenericLiteLLMParams
 
@@ -22,14 +23,14 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM
 
     def validate_anthropic_messages_environment(
         self,
-        headers: dict,
+        headers: dict[str, str],  # mutable-ok: overrides the base signature
         model: str,
-        messages: list[Any],
-        optional_params: dict,
-        litellm_params: dict,
+        messages: list[object],  # mutable-ok: overrides the base signature
+        optional_params: dict[str, object],  # mutable-ok: overrides the base signature
+        litellm_params: dict[str, object],  # mutable-ok: overrides the base signature
         api_key: str | None = None,
         api_base: str | None = None,
-    ) -> tuple[dict, str | None]:
+    ) -> tuple[dict[str, str], str | None]:  # mutable-ok: overrides the base signature
         workspace_id: Final = self._get_workspace_id(optional_params, litellm_params)
         if workspace_id is None:
             raise litellm.AuthenticationError(
@@ -66,6 +67,27 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM
             api_base,
         )
 
+    async def avalidate_anthropic_messages_environment(
+        self,
+        headers: dict[str, str],  # mutable-ok: mirrors validate_anthropic_messages_environment
+        model: str,
+        messages: list[object],  # mutable-ok: mirrors validate_anthropic_messages_environment
+        optional_params: dict[str, object],  # mutable-ok: mirrors validate_anthropic_messages_environment
+        litellm_params: dict[str, object],  # mutable-ok: mirrors validate_anthropic_messages_environment
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> tuple[dict[str, str], str | None]:  # mutable-ok: mirrors validate_anthropic_messages_environment
+        return await run_aws_signing(
+            self.validate_anthropic_messages_environment,
+            headers=headers,
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            api_key=api_key,
+            api_base=api_base,
+        )
+
     def transform_anthropic_messages_request(
         self,
         model: str,

litellm/llms/bedrock/count_tokens/handler.py

@@ -48,7 +48,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
             verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model)
 
             # Get AWS region using existing LiteLLM function
-            aws_region_name: Final = self.get_aws_region_name(
+            aws_region_name: Final = await run_aws_signing(
+                self.get_aws_region_name,
                 optional_params=litellm_params,
                 model=resolved_model,
                 model_id=None,
@@ -64,7 +65,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
             # Get endpoint URL using simplified function
             api_base: Final = litellm_params.get("api_base", None)
             aws_bedrock_runtime_endpoint: Final = litellm_params.get("aws_bedrock_runtime_endpoint", None)
-            endpoint_url: Final = self.get_bedrock_count_tokens_endpoint(
+            endpoint_url: Final = await run_aws_signing(
+                self.get_bedrock_count_tokens_endpoint,
                 model=resolved_model,
                 aws_region_name=aws_region_name,
                 api_base=api_base,

litellm/llms/bedrock/count_tokens/mantle_handler.py

@@ -54,10 +54,10 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
     ) -> dict[str, JsonValue]:
         try:
             request: Final = _COUNT_REQUEST.validate_python(request_data)
-            aws_region_name: Final = self.get_aws_region_name(
-                optional_params=litellm_params, model=resolved_model, model_id=None
+            aws_region_name: Final = await run_aws_signing(
+                self.get_aws_region_name, optional_params=litellm_params, model=resolved_model, model_id=None
             )
-            endpoint_url: Final = self.get_mantle_count_tokens_endpoint(aws_region_name)
+            endpoint_url: Final = await run_aws_signing(self.get_mantle_count_tokens_endpoint, aws_region_name)
             body: Final = self._anthropic_config.transform_request_to_count_tokens(
                 model=resolved_model,
                 messages=request["messages"],

litellm/llms/bedrock/realtime/handler.py

@@ -337,10 +337,7 @@ class BedrockRealtime(BaseAWSLLM):
 
         # Get AWS region
         if aws_region_name is None:
-            optional_params: Final = {
-                "aws_region_name": aws_region_name,
-            }
-            aws_region_name = self._get_aws_region_name(optional_params, model)
+            aws_region_name = await run_aws_signing(self.get_aws_region_name, {}, model)
 
         # Get endpoint URL
         if api_base is not None:

litellm/llms/custom_httpx/llm_http_handler.py

@@ -104,7 +104,7 @@ from litellm.llms.base_llm.vector_store_files.transformation import (
     BaseVectorStoreFilesConfig,
 )
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
-from litellm.llms.bedrock.base_aws_llm import SignsRequestsWithAWS, run_aws_signing, sign_request_off_loop_if_aws
+from litellm.llms.bedrock.base_aws_llm import SignsRequestsWithAWS, run_aws_signing, run_off_loop_if_aws
 from litellm.llms.custom_httpx.container_handler import raise_for_error_status
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -2085,14 +2085,18 @@ class BaseLLMHTTPHandler:
             raise ValueError(f"No provider config found for provider: {custom_llm_provider}")
 
         # Validate environment and get headers
-        headers = provider_config.validate_environment(
+        headers = await run_off_loop_if_aws(
+            provider_config,
+            provider_config.validate_environment,
             api_key=api_key,
             api_base=api_base,
             headers=headers or {},
         )
 
         # Transform the request first
-        data: Final = provider_config.transform_search_request(
+        data: Final = await run_off_loop_if_aws(
+            provider_config,
+            provider_config.transform_search_request,
             query=query,
             optional_params=optional_params,
             api_key=api_key,
@@ -2108,7 +2112,7 @@ class BaseLLMHTTPHandler:
             api_key=api_key,
         )
 
-        signed_headers, signed_json_body = await sign_request_off_loop_if_aws(
+        signed_headers, signed_json_body = await run_off_loop_if_aws(
             provider_config,
             provider_config.sign_request,
             headers=headers,
@@ -2236,7 +2240,7 @@ class BaseLLMHTTPHandler:
                     )
                     provider_config.transform_anthropic_messages_request_on_http_error(e=e, request_data=request_body)
                     recovered_errors = (*recovered_errors, e)
-                headers, signed_json_body = await sign_request_off_loop_if_aws(
+                headers, signed_json_body = await run_off_loop_if_aws(
                     provider_config,
                     provider_config.sign_request,
                     headers=headers,
@@ -2405,7 +2409,9 @@ class BaseLLMHTTPHandler:
         logging_obj.model_call_details.update(request_body)
 
         # Make the request
-        request_url: Final = anthropic_messages_provider_config.get_complete_url(
+        request_url: Final = await run_off_loop_if_aws(
+            anthropic_messages_provider_config,
+            anthropic_messages_provider_config.get_complete_url,
             api_base=api_base,
             api_key=api_key,
             model=model,
@@ -2416,7 +2422,7 @@ class BaseLLMHTTPHandler:
             stream=stream,
         )
 
-        headers, signed_json_body = await sign_request_off_loop_if_aws(
+        headers, signed_json_body = await run_off_loop_if_aws(
             anthropic_messages_provider_config,
             anthropic_messages_provider_config.sign_request,
             headers=headers,
@@ -2967,7 +2973,9 @@ class BaseLLMHTTPHandler:
         stream = response_api_optional_request_params.get("stream", False)
         caller_requested_stream: Final = bool(stream or (extra_body or {}).get("stream"))
 
-        api_base: Final = responses_api_provider_config.get_complete_url(
+        api_base: Final = await run_off_loop_if_aws(
+            responses_api_provider_config,
+            responses_api_provider_config.get_complete_url,
             api_base=litellm_params.api_base,
             litellm_params=dict(litellm_params),
         )
@@ -3009,7 +3017,7 @@ class BaseLLMHTTPHandler:
                 fake_stream=fake_stream,
             )
 
-        headers, signed_body = await sign_request_off_loop_if_aws(
+        headers, signed_body = await run_off_loop_if_aws(
             responses_api_provider_config,
             responses_api_provider_config.sign_request,
             headers=validated_headers,
@@ -3856,7 +3864,9 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             api_key=api_key,
         )
-        complete_api_base: Final = provider_config.get_complete_file_url(
+        complete_api_base: Final = await run_off_loop_if_aws(
+            provider_config,
+            provider_config.get_complete_file_url,
             api_base=api_base,
             api_key=api_key,
             model="",
@@ -3867,7 +3877,9 @@ class BaseLLMHTTPHandler:
         if not complete_api_base:
             raise ValueError("api_base is required for create_file")
         return await self.async_create_file(
-            transformed_request=provider_config.transform_create_file_request(
+            transformed_request=await run_off_loop_if_aws(
+                provider_config,
+                provider_config.transform_create_file_request,
                 model="",
                 create_file_data=create_file_data,
                 litellm_params=litellm_params,
@@ -4334,7 +4346,9 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             api_key=api_key,
         )
-        complete_api_base: Final = provider_config.get_complete_batch_url(
+        complete_api_base: Final = await run_off_loop_if_aws(
+            provider_config,
+            provider_config.get_complete_batch_url,
             api_base=api_base,
             api_key=api_key,
             model=model,
@@ -4345,7 +4359,9 @@ class BaseLLMHTTPHandler:
         if not complete_api_base:
             raise ValueError("api_base is required for create_batch")
         return await self.async_create_batch(
-            transformed_request=provider_config.transform_create_batch_request(
+            transformed_request=await run_off_loop_if_aws(
+                provider_config,
+                provider_config.transform_create_batch_request,
                 model=model,
                 create_batch_data=create_batch_data,
                 litellm_params=litellm_params,
@@ -4818,7 +4834,9 @@ class BaseLLMHTTPHandler:
             **(extra_headers or {}),
         }
 
-        api_base: Final = responses_api_provider_config.get_complete_url(
+        api_base: Final = await run_off_loop_if_aws(
+            responses_api_provider_config,
+            responses_api_provider_config.get_complete_url,
             api_base=litellm_params.api_base,
             litellm_params=dict(litellm_params),
         )
@@ -4836,7 +4854,7 @@ class BaseLLMHTTPHandler:
         )
         data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
 
-        headers, signed_body = await sign_request_off_loop_if_aws(
+        headers, signed_body = await run_off_loop_if_aws(
             responses_api_provider_config,
             responses_api_provider_config.sign_request,
             headers=validated_headers,
@@ -4962,7 +4980,9 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Get URL and params from provider config
-        url, params = provider_config.transform_retrieve_file_request(
+        url, params = await run_off_loop_if_aws(
+            provider_config,
+            provider_config.transform_retrieve_file_request,
             file_id=file_id,
             optional_params={},
             litellm_params=litellm_params,
@@ -5089,7 +5109,9 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Get URL and params from provider config
-        url, params = provider_config.transform_delete_file_request(
+        url, params = await run_off_loop_if_aws(
+            provider_config,
+            provider_config.transform_delete_file_request,
             file_id=file_id,
             optional_params={},
             litellm_params=litellm_params,
@@ -5214,7 +5236,9 @@ class BaseLLMHTTPHandler:
             async_httpx_client = client
 
         # Get URL and params from provider config
-        url, params = provider_config.transform_list_files_request(
+        url, params = await run_off_loop_if_aws(
+            provider_config,
+            provider_config.transform_list_files_request,
             purpose=purpose,
             optional_params={},
             litellm_params=litellm_params,
@@ -5298,7 +5322,9 @@ class BaseLLMHTTPHandler:
             )
             yield page_files[: MAX_FILE_LIST_LIMIT - listed_count]
             listed_count += len(page_files)
-            next_request = self._next_listing_request(latest_page, provider_config, litellm_params, listed_count)
+            next_request = await run_off_loop_if_aws(
+                provider_config, self._next_listing_request, latest_page, provider_config, litellm_params, listed_count
+            )
             if next_request is None:
                 return
             url, params = next_request
@@ -5524,7 +5550,9 @@ class BaseLLMHTTPHandler:
         headers: dict,
         logging_obj: LiteLLMLoggingObj,
     ) -> "_PreparedFileContentRequest":
-        url, params = provider_config.transform_file_content_request(
+        url, params = await run_off_loop_if_aws(
+            provider_config,
+            provider_config.transform_file_content_request,
             file_content_request=file_content_request,
             optional_params={},
             litellm_params=litellm_params,
@@ -10332,7 +10360,9 @@ class BaseLLMHTTPHandler:
         if extra_headers:
             headers.update(extra_headers)
 
-        api_base: Final = vector_store_provider_config.get_complete_url(
+        api_base: Final = await run_off_loop_if_aws(
+            vector_store_provider_config,
+            vector_store_provider_config.get_complete_url,
             api_base=litellm_params.api_base,
             litellm_params=dict(litellm_params),
         )
@@ -10366,7 +10396,7 @@ class BaseLLMHTTPHandler:
             )
         all_optional_params: Final[dict[str, object]] = dict(litellm_params)
         all_optional_params.update(vector_store_search_optional_params or {})
-        headers, signed_json_body = await sign_request_off_loop_if_aws(
+        headers, signed_json_body = await run_off_loop_if_aws(
             vector_store_provider_config,
             vector_store_provider_config.sign_request,
             headers=headers,

litellm/main.py

@@ -400,7 +400,11 @@ class LiteLLM:
 
 
 class Chat:
-    def __init__(self, params, router_obj: "Router | None"):
+    def __init__(
+        self,
+        params: dict[str, object],  # mutable-ok: pops the acompletion flag in place
+        router_obj: "Router | None",
+    ):
         self.params = params
         if self.params.get("acompletion", False) is True:
             self.params.pop("acompletion")
@@ -480,6 +484,7 @@ async def acompletion(
     service_tier: str | None = None,
     store: bool | None = None,
     prompt_cache_key: str | None = None,
+    prompt_cache_retention: str | None = None,
     # set api_base, api_version, api_key
     base_url: str | None = None,
     api_version: str | None = None,
@@ -657,6 +662,7 @@ async def acompletion(
         "service_tier": service_tier,
         "store": store,
         "prompt_cache_key": prompt_cache_key,
+        "prompt_cache_retention": prompt_cache_retention,
         "extra_headers": extra_headers,
         "acompletion": True,  # assuming this is a required parameter
         "thinking": thinking,
@@ -5356,6 +5362,7 @@ def completion(
     service_tier: str | None = None,
     store: bool | None = None,
     prompt_cache_key: str | None = None,
+    prompt_cache_retention: str | None = None,
     # soon to be deprecated params by OpenAI
     functions: list | None = None,
     function_call: str | None = None,
@@ -5478,6 +5485,7 @@ def completion(
             service_tier=service_tier,
             store=store,
             prompt_cache_key=prompt_cache_key,
+            prompt_cache_retention=prompt_cache_retention,
             base_url=base_url,
             api_version=api_version,
             api_key=api_key,
@@ -5843,6 +5851,7 @@ def completion(
             "service_tier": service_tier,
             "store": store,
             "prompt_cache_key": prompt_cache_key,
+            "prompt_cache_retention": prompt_cache_retention,
             "allowed_openai_params": allowed_openai_params,
             "base_model": base_model,
         }
@@ -9143,7 +9152,7 @@ async def ahealth_check(
 
 ####### HELPER FUNCTIONS ################
 ## Set verbose to true -> ```litellm.set_verbose = True```
-def print_verbose(print_statement):
+def print_verbose(print_statement: object):
     try:
         verbose_logger.debug(print_statement)
         if litellm.set_verbose:

litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

@@ -796,7 +796,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
     # logic becomes shared across providers.
 
     #### CALL HOOKS - proxy only ####
-    def _load_credentials(self, bearer_token: str | None = None):
+    def _load_credentials(self, api_key: str | None = None):
         try:
             from botocore.credentials import Credentials
         except ImportError:
@@ -820,7 +820,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
 
         credentials: Final[Credentials | None] = (
             None
-            if bearer_token is not None
+            if bedrock_bearer_token(api_key) is not None
             else self.get_credentials(
                 aws_access_key_id=aws_access_key_id,
                 aws_secret_access_key=aws_secret_access_key,
@@ -1017,9 +1017,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         start_time: "datetime",
         allow_chunking: bool,
     ) -> BedrockGuardrailResponse:
-        credentials, aws_region_name = await run_aws_signing(
-            self._load_credentials, bearer_token=bedrock_bearer_token(api_key)
-        )
+        credentials, aws_region_name = await run_aws_signing(self._load_credentials, api_key=api_key)
 
         completed_chunk_usages: Final[list[BedrockGuardrailUsage]] = []  # mutable-ok: billed-chunk usage accumulator
         try:
@@ -1960,9 +1958,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
             return BedrockGuardrailResponse()
 
         api_key: Final[str | None] = request_data.get("api_key") if request_data else None
-        credentials, aws_region_name = await run_aws_signing(
-            self._load_credentials, bearer_token=bedrock_bearer_token(api_key)
-        )
+        credentials, aws_region_name = await run_aws_signing(self._load_credentials, api_key=api_key)
         body: Final[dict[str, object]] = {"messages": checks_messages, "checks": self.checks}
 
         prepared_request: Final = await run_aws_signing(

litellm/proxy/hooks/proxy_track_cost_callback.py

@@ -359,7 +359,6 @@ class _ProxyDBLogger(CustomLogger):
             increment_spend_counters,
             proxy_logging_obj,
             update_cache,
-            update_cache_read_keys,
         )
 
         verbose_proxy_logger.debug("INSIDE _PROXY_track_cost_callback")
@@ -454,12 +453,6 @@ class _ProxyDBLogger(CustomLogger):
                         request_tags=tags,
                         model_access_groups=model_access_groups,
                         project_id=project_id,
-                        update_cache_read_keys=update_cache_read_keys(
-                            user_id=user_id,
-                            end_user_id=end_user_id,
-                            tags=tags,
-                            response_cost=response_cost,
-                        ),
                     )
                     if not charged:
                         return
@@ -783,7 +776,6 @@ async def _update_database_and_spend_counters(
     request_tags: list[str] | None = None,
     model_access_groups: Sequence[str] | None = None,
     project_id: str | None = None,
-    update_cache_read_keys: Sequence[str] = (),
 ) -> bool:
     """The reservation is reconciled before the spend is persisted, from its own read. One spend counter batch then
     spans the database write and the counter update, so the post-call counters are read with a single MGET after the
@@ -825,7 +817,6 @@ async def _update_database_and_spend_counters(
             request_tags=request_tags,
             model_access_groups=model_access_groups,
             project_id=project_id,
-            update_cache_read_keys=update_cache_read_keys,
         )
 
 
@@ -846,10 +837,7 @@ async def _update_database_and_spend_counters_in_batch(
     request_tags: list[str] | None,
     model_access_groups: Sequence[str] | None,
     project_id: str | None,
-    update_cache_read_keys: Sequence[str],
 ) -> bool:
-    from litellm.proxy.proxy_server import arm_update_cache_read
-
     try:
         charged: Final = await proxy_logging_obj.db_spend_update_writer.update_database(
             token=user_api_key,
@@ -881,7 +869,6 @@ async def _update_database_and_spend_counters_in_batch(
         await _release_budget_reservation(budget_reservation=budget_reservation)
         return False
 
-    await arm_update_cache_read(update_cache_read_keys)
     try:
         await increment_spend_counters(
             token=user_api_key,

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -1459,7 +1459,6 @@ async def bedrock_proxy_route(
     """
     create_request_copy(request)
 
-    aws_region_name: Final = get_secret_str(secret_name="AWS_REGION_NAME")
     if not _is_bedrock_agent_runtime_route(endpoint=endpoint):
         return await bedrock_llm_proxy_route(
             endpoint=endpoint,
@@ -1474,6 +1473,10 @@ async def bedrock_proxy_route(
             detail="bedrock-agent-runtime pass-through is disabled on this proxy.",
         )
 
+    from litellm.llms.bedrock.base_aws_llm import run_aws_signing, sign_aws_json_post
+    from litellm.llms.bedrock.chat import BedrockConverseLLM
+
+    aws_region_name: Final = await run_aws_signing(get_secret_str, secret_name="AWS_REGION_NAME")
     base_target_url: Final = f"https://bedrock-agent-runtime.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}"
     encoded_endpoint = httpx.URL(endpoint).path
 
@@ -1488,10 +1491,6 @@ async def bedrock_proxy_route(
         path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
     )
 
-    # Add or update query parameters
-    from litellm.llms.bedrock.base_aws_llm import run_aws_signing, sign_aws_json_post
-    from litellm.llms.bedrock.chat import BedrockConverseLLM
-
     bedrock_llm: Final = BedrockConverseLLM()
     # Assuming the body contains JSON data, parse it
     try:

litellm/proxy/proxy_server.py

@@ -146,7 +146,6 @@ from litellm.proxy._types import (
     TransformRequestBody,
     UserAPIKeyAuth,
 )
-from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
 from litellm.proxy.common_utils.callback_utils import (
     is_sensitive_callback_key,
     normalize_callback_names,
@@ -308,10 +307,8 @@ from litellm._logging import (
     verbose_router_logger,
 )
 from litellm.caching.caching import DualCache, RedisCache
-from litellm.caching.dual_cache import DeclaredBatchRead
 from litellm.caching.redis_batch import (
     active_post_call_redis_batch,
-    active_request_redis_batches,
     drain_post_call_redis_batches,
 )
 from litellm.caching.redis_cache import RedisCircuitBreakerOpenError, is_redis_timeout_failure
@@ -4236,73 +4233,61 @@ async def run_spend_counter_pipeline(pending: Sequence[PendingSpendIncrement]) -
     return tuple(float(current_value) for current_value in results or ())
 
 
-def update_cache_read_keys(
-    user_id: str | None,
-    end_user_id: str | None,
-    tags: Sequence[object] | None,
-    response_cost: float | None,
-) -> tuple[str, ...]:
-    if response_cost is None:
-        return ()
-    user_keys: tuple[str, ...] = (user_id, GLOBAL_PROXY_SPEND_CACHE_KEY) if user_id is not None else ()
-    end_user_keys: tuple[str, ...] = (end_user_cache_key(end_user_id),) if end_user_id is not None else ()
-    tag_keys: tuple[str, ...] = tuple(tag_cache_key(tag) for tag in tags or () if isinstance(tag, str) and tag)
-    return user_keys + end_user_keys + tag_keys
+_CachedSpendObjectT = TypeVar(
+    "_CachedSpendObjectT", LiteLLM_UserTable, LiteLLM_EndUserTable, LiteLLM_TeamTableCachedObj, LiteLLM_TagTable
+)
 
 
-_UPDATE_CACHE_PREFETCH_SLOT: Final = "update_cache_read"
-
-
-async def arm_update_cache_read(keys: Sequence[str], cache: DualCache | None = None) -> None:
-    """Declares the ``update_cache`` read on the request pipeline once the spend is persisted, so it rides the same
-    round trip as the post-call spend counter read instead of its own."""
-    request: Final = active_request_redis_batches()
-    target: Final = user_api_key_cache if cache is None else cache
-    if request is None or target.redis_cache is None or not keys:
+def _add_spend_to_locally_cached_object(
+    cache: UserApiKeyCache, key: str, model_type: type[_CachedSpendObjectT], response_cost: float
+) -> None:
+    current: Final = cache.get_cache(key=key, local_only=True, model_type=model_type)
+    if current is None:
         return
-    with service_target(AUTH_OBJECTS_TARGET):
-        request.prefetched[_UPDATE_CACHE_PREFETCH_SLOT] = await target.declare_batch_get(
-            keys, request.batch(target.redis_cache)
-        )
+    cache.set_cache(
+        key=key,
+        value=current.model_copy(update={"spend": (current.spend or 0.0) + response_cost}),
+        local_only=True,
+        model_type=model_type,
+        ttl=get_management_object_ttl(cache),
+    )
 
 
-async def _take_armed_update_cache_read(keys: Sequence[str], cache: DualCache) -> Mapping[str, object] | None:
-    request: Final = active_request_redis_batches()
-    if request is None:
+async def _add_spend_to_shared_global_proxy_spend(
+    cache: UserApiKeyCache, response_cost: float, parent_otel_span: Span | None
+) -> float | None:
+    if cache.redis_cache is None:
         return None
-    armed: Final = request.prefetched.pop(_UPDATE_CACHE_PREFETCH_SLOT, None)
-    if not isinstance(armed, DeclaredBatchRead) or armed.keys != tuple(keys):
-        return None
-    values: Final = await cache.async_resolve_batch_get(armed)
-    return MappingProxyType({key: value for key, value in zip(keys, values) if value is not None})
-
-
-async def _read_update_cache_values(
-    keys: Sequence[str], parent_otel_span: Span | None, cache: DualCache | None = None
-) -> Mapping[str, object]:
-    """One batched read for every object ``update_cache`` refreshes; a failed read leaves them all untouched,
-    exactly as a failed per-object GET left that object untouched."""
-    if not keys:
-        return MappingProxyType({})
-    target: Final = user_api_key_cache if cache is None else cache
     try:
-        armed: Final = await _take_armed_update_cache_read(keys, target)
-        if armed is not None:
-            return armed
-        values: Final = await target.async_batch_get_cache(
-            keys=list(keys), parent_otel_span=parent_otel_span, throttle_redis=False
+        return await cache.redis_cache.async_increment_if_present(
+            key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=response_cost, parent_otel_span=parent_otel_span
         )
     except Exception as e:
         verbose_proxy_logger.warning(
-            "Spend tracking - failed to read cached spend objects. Budget enforcement may use stale spend values. "
-            "keys=%s - %s",
-            keys,
+            "Spend tracking - failed to add spend to the global proxy spend in Redis. "
+            "Budget enforcement may use stale spend values. response_cost=%s - %s",
+            response_cost,
             str(e),
         )
-        return MappingProxyType({})
-    if values is None:
-        return MappingProxyType({})
-    return MappingProxyType({key: value for key, value in zip(keys, values) if value is not None})
+        return None
+
+
+async def _add_spend_to_global_proxy_spend(
+    cache: UserApiKeyCache, response_cost: float, parent_otel_span: Span | None
+) -> None:
+    local_before: Final = cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, local_only=True)
+    shared_total: Final = await _add_spend_to_shared_global_proxy_spend(cache, response_cost, parent_otel_span)
+    local_total: Final = cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, local_only=True)
+    if not isinstance(local_total, (int, float)):
+        return
+    if shared_total is not None and local_total != local_before and shared_total < local_total:
+        return
+    cache.set_cache(
+        key=GLOBAL_PROXY_SPEND_CACHE_KEY,
+        value=local_total + response_cost if shared_total is None else shared_total,
+        local_only=True,
+        ttl=get_management_object_ttl(cache),
+    )
 
 
 async def update_cache(
@@ -4313,6 +4298,7 @@ async def update_cache(
     response_cost: float | None,
     parent_otel_span: Span | None,
     tags: list[str] | None = None,
+    cache: UserApiKeyCache | None = None,
 ):
     """
     Use this to update the cache with new user spend.
@@ -4320,14 +4306,7 @@ async def update_cache(
     Put any alerting logic in here.
     """
 
-    values_to_update_in_cache: Final[list[tuple[str, object]]] = []
-    with service_target(AUTH_OBJECTS_TARGET):
-        cached_values: Final = await _read_update_cache_values(
-            keys=update_cache_read_keys(
-                user_id=user_id, end_user_id=end_user_id, tags=tags, response_cost=response_cost
-            ),
-            parent_otel_span=parent_otel_span,
-        )
+    target: Final = user_api_key_cache if cache is None else cache
 
     ### UPDATE KEY SPEND ###
     async def _update_key_cache(token: str, response_cost: float):
@@ -4337,7 +4316,7 @@ async def update_cache(
         else:
             hashed_token = token
         verbose_proxy_logger.debug("_update_key_cache: hashed_token=%s", hashed_token)
-        existing_spend_obj = await user_api_key_cache.async_get_cache(key=hashed_token, model_type=UserAPIKeyAuth)
+        existing_spend_obj = await target.async_get_cache(key=hashed_token, model_type=UserAPIKeyAuth)
         verbose_proxy_logger.debug("_update_key_cache: existing_spend_obj=%s", existing_spend_obj)
         if existing_spend_obj is None:
             return
@@ -4379,184 +4358,17 @@ async def update_cache(
             )
             # set cooldown on alert
 
-    ### UPDATE USER SPEND ###
-    async def _update_user_cache():
-        ## UPDATE CACHE FOR USER ID + GLOBAL PROXY
-        if response_cost is None:
-            return
-        user_ids: Final = [user_id]
+    def _add_spend(key: str, model_type: type[_CachedSpendObjectT], cost: float) -> None:
         try:
-            for _id in user_ids:
-                # Fetch the existing cost for the given user
-                if _id is None:
-                    continue
-                cached_user = cached_values.get(_id)
-                if cached_user is None:
-                    # do nothing if there is no cache value
-                    return
-                existing_spend_obj = CacheCodec.deserialize(cached_user, LiteLLM_UserTable)
-                if existing_spend_obj is None:
-                    return
-                verbose_proxy_logger.debug(
-                    "_update_user_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost
-                )
-
-                existing_spend = existing_spend_obj.spend or 0.0
-                # Calculate the new cost by adding the existing cost and response_cost
-                new_spend = existing_spend + response_cost
-
-                existing_spend_obj.spend = new_spend
-                values_to_update_in_cache.append(
-                    (
-                        _id,
-                        CacheCodec.serialize(existing_spend_obj, model_type=LiteLLM_UserTable),
-                    )
-                )
-            ## UPDATE GLOBAL PROXY ##
-            global_proxy_spend: Final = cached_values.get(GLOBAL_PROXY_SPEND_CACHE_KEY)
-            if not isinstance(global_proxy_spend, (int, float)):
-                # do nothing if not in cache
-                return
-            elif response_cost is not None:
-                increment: Final = global_proxy_spend + response_cost
-                values_to_update_in_cache.append((GLOBAL_PROXY_SPEND_CACHE_KEY, increment))
+            _add_spend_to_locally_cached_object(target, key, model_type, cost)
         except Exception as e:
             verbose_proxy_logger.warning(
-                "Spend tracking - failed to update user spend in cache. "
+                "Spend tracking - failed to update %s spend in cache. "
                 "Budget enforcement may use stale spend values. "
-                "user_id=%s, response_cost=%s - %s\n%s",
-                user_id,
-                response_cost,
-                str(e),
-                traceback.format_exc(),
-            )
-
-    ### UPDATE END-USER SPEND ###
-    async def _update_end_user_cache():
-        if end_user_id is None or response_cost is None:
-            return
-
-        _id: Final = end_user_cache_key(end_user_id)
-        try:
-            # Fetch the existing cost for the given user
-            cached_end_user: Final = cached_values.get(_id)
-            if cached_end_user is None:
-                # if user does not exist in LiteLLM_UserTable, create a new user
-                # do nothing if end-user not in api key cache
-                return
-            existing_spend_obj: Final = CacheCodec.deserialize(cached_end_user, LiteLLM_EndUserTable)
-            if existing_spend_obj is None:
-                return
-            verbose_proxy_logger.debug(
-                "_update_end_user_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost
-            )
-
-            existing_spend: Final = existing_spend_obj.spend or 0.0
-            # Calculate the new cost by adding the existing cost and response_cost
-            new_spend: Final = existing_spend + response_cost
-
-            existing_spend_obj.spend = new_spend
-            values_to_update_in_cache.append(
-                (
-                    _id,
-                    CacheCodec.serialize(existing_spend_obj, model_type=LiteLLM_EndUserTable),
-                )
-            )
-        except Exception as e:
-            verbose_proxy_logger.warning(
-                "Spend tracking - failed to update end user spend in cache. "
-                "Budget enforcement may use stale spend values. "
-                "end_user_id=%s, response_cost=%s - %s\n%s",
-                end_user_id,
-                response_cost,
-                str(e),
-                traceback.format_exc(),
-            )
-
-    ### UPDATE TEAM SPEND ###
-    def _update_team_cache():
-        if team_id is None or response_cost is None:
-            return
-
-        _id: Final = f"team_id:{team_id}"
-        try:
-            current_team: Final = CacheCodec.deserialize(
-                user_api_key_cache.get_cache(key=_id, local_only=True), LiteLLM_TeamTableCachedObj
-            )
-            if current_team is None:
-                return
-            verbose_proxy_logger.debug(
-                "_update_team_db: existing spend: %s; response_cost: %s", current_team, response_cost
-            )
-            user_api_key_cache.set_cache(
-                key=_id,
-                value=CacheCodec.serialize(
-                    current_team.model_copy(update={"spend": (current_team.spend or 0.0) + response_cost}),
-                    model_type=LiteLLM_TeamTableCachedObj,
-                ),
-                local_only=True,
-                ttl=get_management_object_ttl(user_api_key_cache),
-            )
-        except Exception as e:
-            verbose_proxy_logger.warning(
-                "Spend tracking - failed to update team spend in cache. "
-                "Budget enforcement may use stale spend values. "
-                "team_id=%s, response_cost=%s - %s\n%s",
-                team_id,
-                response_cost,
-                str(e),
-                traceback.format_exc(),
-            )
-
-    ### UPDATE TAG SPEND ###
-    async def _update_tag_cache():
-        """
-        Update the tag cache with the new spend.
-        """
-        if tags is None or response_cost is None:
-            return
-
-        try:
-            for tag_name in tags:
-                if not tag_name or not isinstance(tag_name, str):
-                    continue
-
-                cache_key = tag_cache_key(tag_name)
-                # Fetch the existing tag object from cache
-                cached_tag = cached_values.get(cache_key)
-                if cached_tag is None:
-                    # do nothing if tag not in api key cache
-                    continue
-
-                existing_tag_obj = CacheCodec.deserialize(cached_tag, LiteLLM_TagTable)
-                if existing_tag_obj is None:
-                    continue
-
-                verbose_proxy_logger.debug(
-                    "_update_tag_cache: existing spend for tag=%s: %s; response_cost: %s",
-                    tag_name,
-                    existing_tag_obj,
-                    response_cost,
-                )
-
-                existing_spend = existing_tag_obj.spend or 0.0
-                # Calculate the new cost by adding the existing cost and response_cost
-                new_spend = existing_spend + response_cost
-
-                existing_tag_obj.spend = new_spend
-                values_to_update_in_cache.append(
-                    (
-                        cache_key,
-                        CacheCodec.serialize(existing_tag_obj, model_type=LiteLLM_TagTable),
-                    )
-                )
-        except Exception as e:
-            verbose_proxy_logger.warning(
-                "Spend tracking - failed to update tag spend in cache. "
-                "Budget enforcement may use stale spend values. "
-                "tags=%s, response_cost=%s - %s\n%s",
-                tags,
-                response_cost,
+                "key=%s, response_cost=%s - %s\n%s",
+                model_type.__name__,
+                key,
+                cost,
                 str(e),
                 traceback.format_exc(),
             )
@@ -4564,42 +4376,23 @@ async def update_cache(
     with service_target(AUTH_OBJECTS_TARGET):
         if token is not None and response_cost is not None:
             await _update_key_cache(token=token, response_cost=response_cost)
+    if response_cost is None:
+        return
 
-        if user_id is not None:
-            await _update_user_cache()
+    if user_id is not None:
+        _add_spend(user_id, LiteLLM_UserTable, response_cost)
+    if end_user_id is not None:
+        _add_spend(end_user_cache_key(end_user_id), LiteLLM_EndUserTable, response_cost)
+    if team_id is not None:
+        _add_spend(f"team_id:{team_id}", LiteLLM_TeamTableCachedObj, response_cost)
+    for tag_name in tags or ():
+        if tag_name and isinstance(tag_name, str):
+            _add_spend(tag_cache_key(tag_name), LiteLLM_TagTable, response_cost)
 
-        if end_user_id is not None:
-            await _update_end_user_cache()
-
-        if team_id is not None:
-            _update_team_cache()
-
-        if tags is not None:
-            await _update_tag_cache()
-
-    global_proxy_spend_key: Final = GLOBAL_PROXY_SPEND_CACHE_KEY
-    local_object_updates: Final = tuple((k, v) for k, v in values_to_update_in_cache if k != global_proxy_spend_key)
-    shared_scalar_updates: Final = tuple((k, v) for k, v in values_to_update_in_cache if k == global_proxy_spend_key)
-
-    if local_object_updates:
-        with service_target(AUTH_OBJECTS_TARGET):
-            asyncio.create_task(
-                user_api_key_cache.async_set_cache_pipeline(
-                    cache_list=list(local_object_updates),
-                    ttl=get_management_object_ttl(user_api_key_cache),
-                    litellm_parent_otel_span=parent_otel_span,
-                    local_only=True,
-                )
-            )
-    if shared_scalar_updates:
-        with service_target(SPEND_COUNTERS_TARGET):
-            asyncio.create_task(
-                user_api_key_cache.async_set_cache_pipeline(
-                    cache_list=list(shared_scalar_updates),
-                    ttl=get_management_object_ttl(user_api_key_cache),
-                    litellm_parent_otel_span=parent_otel_span,
-                )
-            )
+    if litellm.max_budget <= 0:
+        return
+    with service_target(SPEND_COUNTERS_TARGET):
+        await _add_spend_to_global_proxy_spend(target, response_cost, parent_otel_span)
 
 
 def run_ollama_serve():

tests/e2e/coverage_registry/llm_conversational.yaml

@@ -13,6 +13,8 @@
 - {id: llm.chat_completions.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "gpt-4o vision; high usage"}
 - {id: llm.chat_completions.openai.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching cost optimization"}
 - {id: llm.chat_completions.openai.service_tier.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: nonstream, assertions: [works], source: "OpenAI service_tier param", rationale: "OpenAI scale-tier request option is forwarded and echoed"}
+- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention sent through the proxy, litellm.completion or Router.completion reaches the body sent to OpenAI /v1/chat/completions, so a caller asking for 24h prompt caching gets it instead of a silent drop"}
+- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention on an openai/responses/ model (chat completions in, Responses out) reaches the body sent to OpenAI /v1/responses"}
 - {id: llm.chat_completions.openai.service_tier.stream.echoes_served_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: stream, assertions: [works], source: "litellm_core_utils/streaming_handler.py", fail_before_fix: proven, rationale: "Every relayed stream chunk carries the service_tier OpenAI stamped on it, so a streaming caller can see which tier served the request"}
 - {id: llm.chat_completions.openai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "o-series reasoning; emerging"}
 - {id: llm.chat_completions.openai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "response_schema extraction"}

tests/e2e/coverage_registry/schema.py

@@ -81,6 +81,7 @@ LlmCapability = Literal[
     "pdf_input",
     "logprobs",
     "prompt_cache_1h",
+    "prompt_cache_24h",
     "prompt_cache_5m",
     "response_headers",
     "service_tier",

tests/e2e/llm_translation/test_provider_features_e2e.py

@@ -1,4 +1,4 @@
-"""Live e2e for model-specific request features: service_tier.
+"""Live e2e for model-specific request features: service_tier and prompt_cache_retention.
 
 Each case asserts the feature took effect, not just a 200.
 
@@ -11,26 +11,38 @@ avoided here because it is capacity-constrained and returns a transient 429 when
 flex resources are unavailable. Bedrock and Vertex do not accept service_tier, so
 that cell is OpenAI-only by design.
 
-Prompt caching lives in test_cache_control.py.
+prompt_cache_retention is judged on the body OpenAI receives, captured by a live
+provider edge, not on the reply: OpenAI echoes "24h" on gpt-5.5 even when the
+request never carried the param, so an echo would pass with the param dropped.
+
+Prompt caching hits and misses live in test_cache_control.py.
 """
 
 from __future__ import annotations
 
+import queue
+from collections.abc import Mapping
+from dataclasses import dataclass, field
 from typing import Final
 
 import pytest
+from pydantic import BaseModel
 
-from e2e_config import unique_marker
+from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker
 from e2e_http import unwrap
 from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
 from lifecycle import ResourceManager
 from models import ChatBody, ChatMessage, LiteLLMParamsBody
 from passthrough_client import PassthroughClient
+from provider_edge import LiveEdge, start_provider_edge
+from proxy_client import ProxyClient
 
 pytestmark = pytest.mark.e2e
 
 SERVICE_TIER = "priority"
 OPENAI_BACKEND: Final = "openai/gpt-5.5"
+OPENAI_RESPONSES_BRIDGE_BACKEND: Final = "openai/responses/gpt-5.5"
+PROMPT_CACHE_RETENTION: Final = "24h"
 
 
 class TestServiceTier:
@@ -74,3 +86,153 @@ class TestServiceTier:
             f"service_tier not honored: sent {SERVICE_TIER!r}, response reported "
             f"{response.service_tier!r} ({response})"
         )
+
+
+class _OutboundRetention(BaseModel, frozen=True):
+    prompt_cache_retention: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class _OpenAIBodyCapture:
+    """The raw bodies sent to one OpenAI path, seen by a live edge in front of
+    api.openai.com. Pass ``observe`` as the edge's ``observe_request``."""
+
+    path: str
+    _bodies: queue.SimpleQueue[bytes] = field(default_factory=queue.SimpleQueue)
+
+    def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None:
+        if body is not None and url.split("?")[0].endswith(self.path):
+            self._bodies.put(body)
+
+    def retentions(self) -> tuple[str | None, ...]:
+        bodies: Final = tuple(self._bodies.get_nowait() for _ in range(self._bodies.qsize()))
+        return tuple(_OutboundRetention.model_validate_json(body).prompt_cache_retention for body in bodies)
+
+
+def _openai_edge_api_base(resources: ResourceManager, capture: _OpenAIBodyCapture, *, reachable_from_proxy: bool) -> str:
+    """The proxy may run in a container, so its edge binds where the proxy can
+    reach it; an in-process SDK call reaches the edge on loopback."""
+    edge: Final = (
+        start_provider_edge(
+            LiveEdge(observe_request=capture.observe),
+            bind_host=PROVIDER_EDGE_BIND_HOST,
+            advertise_host=PROVIDER_EDGE_ADVERTISE_HOST,
+        )
+        if reachable_from_proxy
+        else start_provider_edge(LiveEdge(observe_request=capture.observe))
+    )
+    resources.defer(edge.shutdown)
+    return f"{edge.edge.api_base('openai')}/v1"
+
+
+def _retention_backends() -> tuple[object, ...]:
+    """gpt-5.5 straight to /v1/chat/completions, and the openai/responses/ bridge
+    (chat completions in, Responses out) to /v1/responses."""
+    return (
+        pytest.param(
+            OPENAI_BACKEND,
+            "/chat/completions",
+            marks=(
+                pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention"),
+                meta(
+                    Subject(
+                        domain=Domain.LLM_TRANSLATION,
+                        route=Route.CHAT_COMPLETIONS,
+                        providers=(Provider.OPENAI,),
+                        models=(OPENAI_BACKEND,),
+                        mode=Mode.NONSTREAM,
+                    )
+                ),
+            ),
+            id="chat_completions",
+        ),
+        pytest.param(
+            OPENAI_RESPONSES_BRIDGE_BACKEND,
+            "/responses",
+            marks=(
+                pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention"),
+                meta(
+                    Subject(
+                        domain=Domain.LLM_TRANSLATION,
+                        route=Route.CHAT_COMPLETIONS,
+                        providers=(Provider.OPENAI,),
+                        models=(OPENAI_RESPONSES_BRIDGE_BACKEND,),
+                        mode=Mode.NONSTREAM,
+                    )
+                ),
+            ),
+            id="responses_bridge",
+        ),
+    )
+
+
+class TestPromptCacheRetention:
+    @pytest.mark.provider_edge_host
+    @pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
+    def test_proxy_chat_completions_forwards_prompt_cache_retention_to_openai(
+        self, proxy: ProxyClient, resources: ResourceManager, backend: str, openai_path: str
+    ) -> None:
+        capture: Final = _OpenAIBodyCapture(openai_path)
+        api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=True)
+        model: Final = f"e2e-cache-retention-{unique_marker()}"
+        model_id: Final = proxy.create_model(
+            model, LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY", api_base=api_base)
+        )
+        resources.defer(lambda: proxy.delete_model(model_id))
+
+        unwrap(
+            proxy.chat(
+                resources.key(),
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content="reply with one word")],
+                    prompt_cache_retention=PROMPT_CACHE_RETENTION,
+                ),
+            )
+        )
+
+        assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
+            f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
+        )
+
+    @pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
+    def test_sdk_completion_forwards_prompt_cache_retention_to_openai(
+        self, resources: ResourceManager, backend: str, openai_path: str
+    ) -> None:
+        import litellm
+
+        capture: Final = _OpenAIBodyCapture(openai_path)
+        api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False)
+
+        litellm.completion(
+            model=backend,
+            messages=[{"role": "user", "content": "reply with one word"}],
+            api_base=api_base,
+            prompt_cache_retention=PROMPT_CACHE_RETENTION,
+        )
+
+        assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
+            f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
+        )
+
+    @pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
+    def test_router_completion_forwards_prompt_cache_retention_to_openai(
+        self, resources: ResourceManager, backend: str, openai_path: str
+    ) -> None:
+        import litellm
+
+        capture: Final = _OpenAIBodyCapture(openai_path)
+        api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False)
+        router: Final = litellm.Router(
+            model_list=[{"model_name": "cache-retention", "litellm_params": {"model": backend, "api_base": api_base}}]
+        )
+
+        router.completion(  # pyright: ignore[reportUnknownMemberType]  # Router.completion forwards an untyped **kwargs to litellm.completion
+            model="cache-retention",
+            messages=[{"role": "user", "content": "reply with one word"}],
+            prompt_cache_retention=PROMPT_CACHE_RETENTION,
+        )
+
+        assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
+            f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
+        )

tests/e2e/models.py

@@ -364,6 +364,7 @@ class ChatBody(BaseModel):
     thinking: ThinkingParam | None = None
     service_tier: str | None = None
     prompt_cache_key: str | None = None
+    prompt_cache_retention: str | None = None
     tools: Sequence[ChatTool | McpChatTool | GeminiUrlContextTool] | None = None
     tool_choice: str | None = None
     guardrails: list[str] | None = None

tests/e2e/perf/cases/mixed_size_requests_2rps.py

@@ -62,24 +62,24 @@ CASE: Final = PerformanceTestCase(
     ),
     limits=(
         Limit("unexpected_status_count", max=3),
-        Limit("gateway_added_seconds", percentile=50, max=0.58),
-        Limit("gateway_added_seconds", percentile=99, max=6.1),
-        Limit("litellm_overhead_seconds", percentile=50, max=0.1),
-        Limit("litellm_overhead_seconds", percentile=99, max=3.4),
-        Limit("litellm_response_seconds", percentile=50, max=1.0),
-        Limit("litellm_response_seconds", percentile=99, max=3.8),
-        Limit("prometheus_redis_latency_mean_seconds", max=0.005),
-        Limit("postgres_active_sessions", max=0.016),
-        Limit("postgres_slowest_statement_seconds", max=0.25),
-        Limit("postgres_peak_connections", max=45),
-        Limit("time_to_first_token_seconds", percentile=50, max=2.7),
-        Limit("time_to_first_token_seconds", percentile=99, max=8.8),
+        Limit("gateway_added_seconds", percentile=50, max=0.29),
+        Limit("gateway_added_seconds", percentile=99, max=2.7),
+        Limit("litellm_overhead_seconds", percentile=50, max=0.052),
+        Limit("litellm_overhead_seconds", percentile=99, max=1.8),
+        Limit("litellm_response_seconds", percentile=50, max=0.93),
+        Limit("litellm_response_seconds", percentile=99, max=2.7),
+        Limit("prometheus_redis_latency_mean_seconds", max=0.002),
+        Limit("postgres_active_sessions", max=0.0094),
+        Limit("postgres_slowest_statement_seconds", max=0.042),
+        Limit("postgres_peak_connections", max=23),
+        Limit("time_to_first_token_seconds", percentile=50, max=1.56),
+        Limit("time_to_first_token_seconds", percentile=99, max=4.5),
         Limit("memory_growth_megabytes", max=600),
-        Limit("cpu_seconds_per_request", max=1.26),
-        Limit("postgres_calls_per_request", max=3.5),
-        Limit("postgres_max_statement_per_request", max=0.5),
-        Limit("postgres_max_background_statement_per_request", max=0.9),
-        Limit("redis_calls_per_request", max=21.0),
+        Limit("cpu_seconds_per_request", max=0.57),
+        Limit("postgres_calls_per_request", max=2.64),
+        Limit("postgres_max_statement_per_request", max=0.43),
+        Limit("postgres_max_background_statement_per_request", max=0.86),
+        Limit("redis_calls_per_request", max=15.4),
     ),
     background_cap_exempt_tables=frozenset({"LiteLLM_DailyGuardrailMetrics"}),
 )

tests/e2e/perf/cases/small_bodies_5rps.py

@@ -64,21 +64,21 @@ CASE: Final = PerformanceTestCase(
     ),
     limits=(
         Limit("unexpected_status_count", max=3),
-        Limit("gateway_added_seconds", percentile=50, max=0.29),
-        Limit("litellm_overhead_seconds", percentile=50, max=0.012),
+        Limit("gateway_added_seconds", percentile=50, max=0.165),
+        Limit("litellm_overhead_seconds", percentile=50, max=0.006),
         Limit("litellm_overhead_seconds", percentile=99, max=0.35),
-        Limit("litellm_response_seconds", percentile=50, max=1.0),
-        Limit("litellm_response_seconds", percentile=99, max=2.0),
-        Limit("prometheus_redis_latency_mean_seconds", max=0.006),
-        Limit("postgres_active_sessions", max=0.01),
-        Limit("postgres_slowest_statement_seconds", max=0.025),
-        Limit("postgres_peak_connections", max=45),
-        Limit("time_to_first_token_seconds", percentile=99, max=3.7),
+        Limit("litellm_response_seconds", percentile=50, max=0.91),
+        Limit("litellm_response_seconds", percentile=99, max=1.6),
+        Limit("prometheus_redis_latency_mean_seconds", max=0.0026),
+        Limit("postgres_active_sessions", max=0.0052),
+        Limit("postgres_slowest_statement_seconds", max=0.011),
+        Limit("postgres_peak_connections", max=23),
+        Limit("time_to_first_token_seconds", percentile=99, max=2.05),
         Limit("memory_growth_megabytes", max=100),
-        Limit("postgres_calls_per_request", max=1.8),
-        Limit("postgres_max_statement_per_request", max=0.5),
-        Limit("postgres_max_background_statement_per_request", max=0.9),
-        Limit("redis_calls_per_request", max=18),
-        Limit("cpu_seconds_per_request", max=0.29),
+        Limit("postgres_calls_per_request", max=1.31),
+        Limit("postgres_max_statement_per_request", max=0.27),
+        Limit("postgres_max_background_statement_per_request", max=0.92),
+        Limit("redis_calls_per_request", max=13.8),
+        Limit("cpu_seconds_per_request", max=0.15),
     ),
 )

tests/e2e_harness/perf/test_upgrade.py

@@ -295,19 +295,30 @@ def test_invalid_200_is_not_an_allowed_non_200(failed: WireRecord) -> None:
         _ = Probe().monitor().poll((failed,))
 
 
+def _ceiling(metric: str, percentile: int | None = None) -> float:
+    return next(
+        limit.max for limit in upgrade_case().limits if limit.metric == metric and limit.percentile == percentile
+    )
+
+
 @pytest.mark.parametrize(
-    "failures,overhead,ttft,growth,expected",
+    "over,expected",
     [
-        (3, 0.29, 3.7, 100, ()),
-        (4, 0.29, 3.7, 100, ("unexpected_status_count",)),
-        (3, 0.30, 3.7, 100, ("gateway_added_seconds percentile 50",)),
-        (3, 0.29, 3.8, 100, ("time_to_first_token_seconds percentile 99",)),
-        (3, 0.29, 3.7, 101, ("memory_growth_megabytes",)),
+        ("", ()),
+        ("unexpected_status_count", ("unexpected_status_count",)),
+        ("gateway_added_seconds", ("gateway_added_seconds percentile 50",)),
+        ("time_to_first_token_seconds", ("time_to_first_token_seconds percentile 99",)),
+        ("memory_growth_megabytes", ("memory_growth_megabytes",)),
     ],
 )
 def test_upgrade_verdict_uses_regular_limits_without_cpu_or_datastore_samples(
-    failures: int, overhead: float, ttft: float, growth: float, expected: tuple[str, ...]
+    over: str, expected: tuple[str, ...]
 ) -> None:
+    """Each sample sits exactly on the case's ceiling, so only the one nudged past it goes over."""
+    failures: Final = int(_ceiling("unexpected_status_count")) + (over == "unexpected_status_count")
+    overhead: Final = _ceiling("gateway_added_seconds", 50) * (1.01 if over == "gateway_added_seconds" else 1)
+    ttft: Final = _ceiling("time_to_first_token_seconds", 99) * (1.01 if over == "time_to_first_token_seconds" else 1)
+    growth: Final = _ceiling("memory_growth_megabytes") + (over == "memory_growth_megabytes")
     nonstream: Final = tuple(record(i).model_copy(update={"total_s": overhead, "mock_delay_s": 0}) for i in range(100))
     stream: Final = record(100).model_copy(update={"stream": True, "first_token_s": ttft})
     observed: Final = Observation(

tests/integration/authorization/test_agent_least_privilege.py

@@ -8,7 +8,6 @@ import httpx
 import pytest
 import yaml
 from integration._support.client import Gateway, eventually, object_value, string_value
-from integration._support.database import read_rows
 from integration._support.process import owned_proxy
 from integration._support.wire import Reply, Request, Wire, wire_server
 
@@ -108,7 +107,7 @@ def _create_agent(candidate: Gateway, wire: Wire, agent_name: str, cleanups: Exi
 def _delete_agent(candidate: Gateway, agent_id: str) -> None:
     deleted: Final = candidate.request("DELETE", f"/v1/agents/{agent_id}")
     assert deleted.status_code == 200, deleted.text
-    assert read_rows('SELECT agent_id FROM "LiteLLM_AgentsTable" WHERE agent_id=%s', (agent_id,)) == []
+    assert candidate.rows('SELECT agent_id FROM "LiteLLM_AgentsTable" WHERE agent_id=%s', (agent_id,)) == []
 
 
 def _card(candidate: Gateway, agent_id: str, key: str) -> httpx.Response:

tests/integration/cost_calculation/cases/anthropic.py

@@ -4478,6 +4478,7 @@ CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
                 "output_tokens_details": {"thinking_tokens": 0},
                 "service_tier": "standard",
                 "inference_geo": "global",
+                "speed": "standard",
             },
             "diagnostics": None,
         },
@@ -4502,7 +4503,7 @@ CLAUDE_SONNET_5_5_MESSAGES_STREAM: Final = CostTrackingTestCase(
     response=SseResponse(
         content_type="text/event-stream",
         frames=(
-            'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-sonnet-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}',
+            'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-sonnet-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"diagnostics":null}}',
             'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}',
             'event: ping\ndata: {"type":"ping"}',
             'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}}',
@@ -4877,6 +4878,7 @@ CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
                 "output_tokens_details": {"thinking_tokens": 0},
                 "service_tier": "standard",
                 "inference_geo": "global",
+                "speed": "standard",
             },
             "diagnostics": None,
         },
@@ -4901,7 +4903,7 @@ CLAUDE_HAIKU_5_5_MESSAGES_STREAM: Final = CostTrackingTestCase(
     response=SseResponse(
         content_type="text/event-stream",
         frames=(
-            'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-haiku-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}',
+            'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-haiku-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"diagnostics":null}}',
             'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}',
             'event: ping\ndata: {"type":"ping"}',
             'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}}',
@@ -5639,6 +5641,7 @@ CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
                 "output_tokens_details": {"thinking_tokens": 0},
                 "service_tier": "standard",
                 "inference_geo": "global",
+                "speed": "standard",
             },
             "diagnostics": None,
         },
@@ -5663,7 +5666,7 @@ CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = CostTrackingTestCase(
     response=SseResponse(
         content_type="text/event-stream",
         frames=(
-            'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-fable-5-1","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}',
+            'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-fable-5-1","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"diagnostics":null}}',
             'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
             'event: ping\ndata: {"type":"ping"}',
             'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted "}}',
@@ -5974,6 +5977,7 @@ CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
                 "output_tokens_details": {"thinking_tokens": 0},
                 "service_tier": "standard",
                 "inference_geo": "global",
+                "speed": "standard",
             },
             "diagnostics": None,
         },
@@ -5998,7 +6002,7 @@ CLAUDE_OPUS_5_5_MESSAGES_STREAM: Final = CostTrackingTestCase(
     response=SseResponse(
         content_type="text/event-stream",
         frames=(
-            'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-opus-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}',
+            'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-opus-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"diagnostics":null}}',
             'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}',
             'event: ping\ndata: {"type":"ping"}',
             'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}}',

tests/integration/cost_calculation/catalog.py

@@ -65,6 +65,7 @@ BRIDGE_COST_PARITY_CASES: Final[tuple[BridgeCostParityTestCase, ...]] = (
     *anthropic.BRIDGE_COST_PARITY,
     *gemini.BRIDGE_COST_PARITY,
     *vertex_ai.BRIDGE_COST_PARITY,
+    *misc.BRIDGE_COST_PARITY,
 )
 _PARITY_LEGS: Final = frozenset(leg.name for pair in PARITY_CASES for leg in (pair.plain, pair.streamed)) | frozenset(
     leg.name for pair in BRIDGE_COST_PARITY_CASES for leg in (pair.native, pair.bridged)

tests/integration/cost_calculation/cost_map.py

@@ -597,6 +597,7 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
         "cache_read_input_token_cost_priority": 2.16e-07,
         "google_maps_grounding_cost_per_query": 0.0168,
         "input_cost_per_audio_token": 2.4e-06,
+        "input_cost_per_audio_token_priority": 4.32e-06,
         "input_cost_per_token": 2.4e-06,
         "input_cost_per_token_flex": 1.2e-06,
         "input_cost_per_token_priority": 4.32e-06,

tests/integration/cost_calculation/cost_tracking_case.py

@@ -94,6 +94,7 @@ class CostMapEntry(BaseModel):
     tiered_pricing: tuple[TieredPrice, ...] | None = None
     output_cost_per_reasoning_token: float | None = None
     input_cost_per_audio_token: float | None = None
+    input_cost_per_audio_token_priority: float | None = None
     input_cost_per_second: float | None = None
     output_cost_per_second: float | None = None
     input_cost_per_character: float | None = None

tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py

@@ -1683,8 +1683,8 @@ def test_responses_input_tokens_sends_mantle_the_tool_use_invoke_inference_sends
         wire_server(_mantle(), port=mantle_port) as mantle,
         gateway.scenario() as scenario,
     ):
-        counted_model: Final = _deployment(scenario, runtime.url)
-        invoked_model: Final = _deployment(scenario, runtime.url, f"invoke/{_OPUS}")
+        counted_model: Final = _fallback_deployment(scenario, runtime.url)
+        invoked_model: Final = _fallback_deployment(scenario, runtime.url, f"invoke/{_OPUS}")
         generated: Final = gateway.request(
             "POST", "/v1/responses", {**_FORCED_TOOL_CHOICE, "model": invoked_model, "max_output_tokens": 16}
         )
@@ -1729,7 +1729,7 @@ def test_nova_count_sends_the_tool_specs_converse_inference_sends(counting_proxy
     path, body, count_field = _NOVA_COUNTS[route]
     gateway: Final = counting_proxy.gateway
     with wire_server(_counting_every_model) as runtime, gateway.scenario() as scenario:
-        model: Final = _deployment(scenario, runtime.url, _NOVA)
+        model: Final = _fallback_deployment(scenario, runtime.url, _NOVA)
         generated: Final = gateway.request(
             "POST",
             "/v1/chat/completions",

tests/integration/providers/test_hosted_vllm_stream_stall_wire.py

@@ -42,8 +42,8 @@ _RESPONSES_TARGET: Final = "/v1/responses"
 _STREAM_TIMEOUT_SECONDS: Final = 1
 _DEPLOYMENT_TIMEOUT_SECONDS: Final = 30
 _PAUSE_SECONDS: Final = 3.0
-_HOLD_SECONDS: Final = 30.0
 _ROW_SECONDS: Final = 60
+_HOLD_SECONDS: Final = 2.0 * _ROW_SECONDS
 _EXTRA: Final[dict[str, JsonValue]] = {"num_retries": 0, "cache": {"no-cache": True}}
 _ROWS: Final = 'SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE model_group=%s'
 _EITHER_GROUP_ROWS: Final = 'SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE model_group IN (%s, %s)'

tests/integration/routing/test_responses_websocket_turn_admission.py

@@ -27,7 +27,6 @@ from integration._support.client import (
     object_value,
     string_value,
 )
-from integration._support.database import read_rows
 from integration._support.process import graceful_stop_seconds, owned_proxy
 from integration._support.responses_ws_peer import (
     PROVIDER_MODEL,
@@ -260,9 +259,9 @@ async def _until(read: Callable[[], float], satisfied: Callable[[float], bool])
     return await asyncio.to_thread(eventually, read, satisfied, 15)
 
 
-def _failure_rows(key: str) -> list[dict[str, JsonValue]]:
+def _failure_rows(gateway: Gateway, key: str) -> list[dict[str, JsonValue]]:
     rows: Final = eventually(
-        lambda: read_rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
+        lambda: gateway.rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
         lambda found: any(row["status"] == "failure" for row in found),
         seconds=70,
     )
@@ -291,7 +290,7 @@ def test_a_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_log
         _assert_refused(second)
         _assert_refused(third)
         assert deployment.forwarded_turns() == 1
-        failures: Final = _failure_rows(key)
+        failures: Final = _failure_rows(limited, key)
         assert len(failures) == 1, failures
         assert failures[0]["model_group"] == deployment.name, failures
         assert tag in list_value(failures[0]["request_tags"]), failures
@@ -355,7 +354,7 @@ def test_a_turn_whose_admission_check_fails_gets_a_500_frame_and_the_socket_serv
         _assert_turn_check_failed(third, fault)
         completed(fourth)
         assert deployment.forwarded_turns() == 2
-        failures: Final = _failure_rows(key)
+        failures: Final = _failure_rows(faulting, key)
         assert len(failures) == 1, failures
         assert failures[0]["model_group"] == deployment.name, failures
         recorded: Final = object_value(object_value(failures[0]["metadata"])["error_information"])
@@ -387,7 +386,7 @@ def test_a_turn_still_streaming_when_the_next_turns_check_fails_completes_and_is
         billed: Final = completed(streamed)
         assert deployment.forwarded_turns() == 1
         rows: Final = eventually(
-            lambda: read_rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
+            lambda: faulting.rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
             lambda found: any(row["request_id"] == billed["id"] for row in found),
             seconds=70,
         )

tests/integration/spend/test_global_proxy_spend_counter.py

@@ -0,0 +1,58 @@
+import os
+import uuid
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.caching.redis_cache import RedisCache
+from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
+from litellm.proxy.proxy_server import update_cache
+
+
+async def _add_request_cost(cache: UserApiKeyCache, response_cost: float) -> None:
+    await update_cache(
+        token=None,
+        user_id=None,
+        end_user_id=None,
+        team_id=None,
+        response_cost=response_cost,
+        parent_otel_span=None,
+        cache=cache,
+    )
+
+
+@pytest.mark.asyncio
+async def test_global_proxy_spend_adds_every_instance_cost_and_stays_cleared_after_a_budget_reset(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(litellm, "max_budget", 100.0)
+    redis_cache: Final = RedisCache(
+        host=os.environ["REDIS_HOST"],
+        port=int(os.environ["REDIS_PORT"]),
+        namespace=f"global-proxy-spend-{uuid.uuid4().hex}",
+    )
+    gateway: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache)
+    peer: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache)
+
+    try:
+        await gateway.async_set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0, ttl=60)
+        await _add_request_cost(peer, 0.5)
+        await _add_request_cost(gateway, 0.25)
+        before_reset: Final = (
+            gateway.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, local_only=True),
+            await redis_cache.async_get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY),
+        )
+        ttl_before_reset: Final = await redis_cache.async_get_ttl(key=GLOBAL_PROXY_SPEND_CACHE_KEY)
+
+        await gateway.async_delete_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY)
+        await _add_request_cost(peer, 0.5)
+        await _add_request_cost(gateway, 0.25)
+
+        assert before_reset == (10.75, 10.75)
+        assert ttl_before_reset is not None and 0 < ttl_before_reset <= 60
+        assert await redis_cache.async_get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY) is None
+    finally:
+        await redis_cache.async_delete_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY)

tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py

@@ -12,7 +12,6 @@ import httpx
 import pytest
 import yaml
 from integration._support.client import Gateway, eventually, gateway_from_environment
-from integration._support.database import read_rows
 from integration._support.forward_proxy import tunnelling_forward_proxy
 from integration._support.process import graceful_stop_seconds, owned_proxy
 from integration._support.tls import server_context, write_self_signed_cert
@@ -289,9 +288,9 @@ def gateway_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
         yield _Rig(proxy, upstream)
 
 
-def _spend_row(key: str) -> dict[str, JsonValue]:
+def _spend_row(gateway: Gateway, key: str) -> dict[str, JsonValue]:
     rows: Final = eventually(
-        lambda: read_rows(
+        lambda: gateway.rows(
             'SELECT spend, custom_llm_provider FROM "LiteLLM_SpendLogs" WHERE api_key=%s',
             (sha256(key.encode()).hexdigest(),),
         ),
@@ -301,14 +300,14 @@ def _spend_row(key: str) -> dict[str, JsonValue]:
     return rows[0]
 
 
-def _spend_for_key(key: str) -> float:
-    spend: Final = _spend_row(key)["spend"]
+def _spend_for_key(gateway: Gateway, key: str) -> float:
+    spend: Final = _spend_row(gateway, key)["spend"]
     assert isinstance(spend, float), key
     return spend
 
 
-def _provider_for_key(key: str) -> JsonValue:
-    return _spend_row(key)["custom_llm_provider"]
+def _provider_for_key(gateway: Gateway, key: str) -> JsonValue:
+    return _spend_row(gateway, key)["custom_llm_provider"]
 
 
 def _post_and_read(rig: _Rig, path: str, body: Mapping[str, JsonValue], headers: Mapping[str, str]) -> None:
@@ -347,7 +346,7 @@ def test_claude_on_vertex_passthrough_spend_carries_the_regional_uplift_whether_
                 {**body, "stream": action == "streamRawPredict"},
                 {"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"},
             )
-        spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
+        spend: Final = {cell: _spend_for_key(rig.proxy, key) for cell, key in keys.items()}
     standard: Final = _list_price(f"vertex_ai/{_CLAUDE_ROUTE_MODEL}")
     regional: Final = standard * _uplift(f"vertex_ai/{_CLAUDE_ROUTE_MODEL}")
     assert spend == pytest.approx(
@@ -370,7 +369,7 @@ def test_v1beta_generate_content_on_a_vertex_deployment_bills_the_deployment_loc
         keys: Final = {cell: scenario.key() for cell in cells}
         for (deployment, action), key in keys.items():
             _post_and_read(rig, f"/v1beta/models/{deployment}:{action}", body, {"x-goog-api-key": key})
-        spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
+        spend: Final = {cell: _spend_for_key(rig.proxy, key) for cell, key in keys.items()}
     standard: Final = _list_price(f"vertex_ai/{_GEMINI}")
     regional: Final = standard * _uplift(f"vertex_ai/{_GEMINI}")
     assert spend == pytest.approx(
@@ -404,8 +403,8 @@ def test_gemini_passthrough_bills_a_named_custom_priced_deployment_only_for_keys
             for key in (allowed[action], plain[action]):
                 _post_and_read(rig, f"/gemini/v1beta/models/{_GEMINI}:{action}", body, {"x-goog-api-key": key})
         spend: Final = {
-            **{("allowed", action): _spend_for_key(key) for action, key in allowed.items()},
-            **{("plain", action): _spend_for_key(key) for action, key in plain.items()},
+            **{("allowed", action): _spend_for_key(rig.proxy, key) for action, key in allowed.items()},
+            **{("plain", action): _spend_for_key(rig.proxy, key) for action, key in plain.items()},
         }
     custom_price: Final = _PROMPT_TOKENS * _CUSTOM_INPUT_RATE + _COMPLETION_TOKENS * _CUSTOM_OUTPUT_RATE
     list_price: Final = _list_price(f"gemini/{_GEMINI}")
@@ -441,7 +440,10 @@ def test_openai_passthrough_stream_cut_off_after_its_usage_bills_what_the_finish
             assert response.status_code == 200
             with pytest.raises(httpx.RemoteProtocolError):
                 response.read()
-        spend: Final = {"finished": _spend_for_key(finished_key), "cut off": _spend_for_key(cut_key)}
+        spend: Final = {
+            "finished": _spend_for_key(rig.proxy, finished_key),
+            "cut off": _spend_for_key(rig.proxy, cut_key),
+        }
     list_price: Final = _list_price(_OPENAI_MODEL)
     assert spend == pytest.approx({"finished": list_price, "cut off": list_price})
 
@@ -484,7 +486,7 @@ def test_a_body_base_model_reprices_passthrough_spend_only_for_keys_that_allow_c
                 },
                 _caller_headers(route, key),
             )
-        spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
+        spend: Final = {cell: _spend_for_key(rig.proxy, key) for cell, key in keys.items()}
     forwarded: Final = rig.upstream.drain()
     marked_bodies: Final = {
         cell: tuple(request.body for request in forwarded if marker.encode() in request.body)
@@ -521,7 +523,7 @@ def test_a_non_string_body_base_model_bills_the_list_price_instead_of_nothing(ri
                 {"contents": [{"role": "user", "parts": [{"text": uuid.uuid4().hex}]}], "base_model": 123},
                 _caller_headers(route, key),
             )
-        spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
+        spend: Final = {cell: _spend_for_key(rig.proxy, key) for cell, key in keys.items()}
     list_price: Final = {
         "vertex_ai": _list_price(f"vertex_ai/{_GEMINI}"),
         "gemini": _list_price(f"gemini/{_GEMINI}"),
@@ -538,8 +540,8 @@ def test_gemini_passthrough_through_a_gemini_api_base_gateway_logs_and_bills_as_
         keys: Final = {action: scenario.key() for action in _GEMINI_ACTIONS}
         for action, key in keys.items():
             _post_and_read(gateway_rig, f"/gemini/v1beta/models/{_GEMINI}:{action}", body, {"x-goog-api-key": key})
-        providers: Final = {action: _provider_for_key(key) for action, key in keys.items()}
-        spend: Final = {action: _spend_for_key(key) for action, key in keys.items()}
+        providers: Final = {action: _provider_for_key(gateway_rig.proxy, key) for action, key in keys.items()}
+        spend: Final = {action: _spend_for_key(gateway_rig.proxy, key) for action, key in keys.items()}
     forwarded: Final = tuple(
         path
         for path in (urlsplit(request.target).path for request in gateway_rig.upstream.drain())
@@ -570,8 +572,8 @@ def test_a_configured_passthrough_to_a_non_google_gateway_bills_as_vertex_whatev
                 body,
                 {"Authorization": f"Bearer {key}"},
             )
-        providers: Final = {cell: _provider_for_key(key) for cell, key in keys.items()}
-        spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
+        providers: Final = {cell: _provider_for_key(gateway_rig.proxy, key) for cell, key in keys.items()}
+        spend: Final = {cell: _spend_for_key(gateway_rig.proxy, key) for cell, key in keys.items()}
     short_path_spend: Final = {action: spend[("short gateway path", action)] for action in _GEMINI_ACTIONS}
     assert providers == {cell: "vertex_ai" for cell in cells}
     assert {action: spend[("vertex resource", action)] for action in _GEMINI_ACTIONS} == pytest.approx(

tests/integration/spend/test_vertex_passthrough_regional_uplift_spend.py

@@ -10,7 +10,6 @@ import httpx
 import pytest
 import yaml
 from integration._support.client import Gateway, eventually, gateway_from_environment
-from integration._support.database import read_rows
 from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy
 from integration._support.process import graceful_stop_seconds, owned_proxy
 from integration._support.tls import server_context, write_self_signed_cert
@@ -150,12 +149,12 @@ def _spend(rig: _Rig, key: str, location: str, action: str, model: str = _MODEL)
         body: Final = response.read()
         assert response.status_code == 200, body
         call_id: Final = response.headers["x-litellm-call-id"]
-    return _logged_spend(call_id)
+    return _logged_spend(rig.proxy, call_id)
 
 
-def _logged_spend(call_id: str) -> float:
+def _logged_spend(gateway: Gateway, call_id: str) -> float:
     rows: Final = eventually(
-        lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)),
+        lambda: gateway.rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)),
         lambda found: len(found) == 1,
         seconds=70,
     )
@@ -193,7 +192,7 @@ def test_custom_priced_vertex_passthrough_bills_the_deployment_price_even_when_t
             call_id: Final = response.headers["x-litellm-call-id"]
             with pytest.raises(httpx.RemoteProtocolError):
                 response.read()
-        cut_off: Final = _logged_spend(call_id)
+        cut_off: Final = _logged_spend(rig.proxy, call_id)
     custom_price: Final = _PROMPT_TOKENS * _CUSTOM_INPUT_RATE + _COMPLETION_TOKENS * _CUSTOM_OUTPUT_RATE
     assert {**finished, "cut off": cut_off} == pytest.approx(
         {_GENERATE: custom_price, _STREAM: custom_price, "cut off": custom_price}

tests/unit/caching/test_request_redis_batch_post_call.py

@@ -10,31 +10,27 @@ import datetime
 import hashlib
 import json
 from collections.abc import Awaitable, Callable, Mapping, Sequence
-from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
 import litellm
-from litellm._internal_context import current_service_target
 from litellm.caching.caching import Cache
 from litellm.caching.dual_cache import DualCache
 from litellm.caching.in_memory_cache import InMemoryCache
 from litellm.caching.redis_batch import (
     active_post_call_redis_batch,
-    active_request_redis_batches,
     drain_post_call_redis_batches,
     flush_post_call_redis_batches,
     request_redis_batch_scope,
 )
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
-from litellm.proxy.auth.auth_object_prefetch import AUTH_OBJECTS_TARGET
 from litellm.proxy.hooks.parallel_request_limiter_v3 import (
     PARALLEL_RELEASE_SCRIPT,
     TOKEN_INCREMENT_SCRIPT,
     ParallelSlotAcquisition,
-    RequestRateLimiterStash,
     PROXY_MaxParallelRequestsHandler_v3,
+    RequestRateLimiterStash,
 )
 from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement
 from litellm.proxy.utils import InternalUsageCache
@@ -518,140 +514,3 @@ async def test_a_cancelled_post_call_flush_keeps_the_shared_spend_counter_and_co
     assert redis_cache.alone == [], "a cancel says nothing about the shared counter, so Redis keeps it"
     assert spend_cache.in_memory_cache.get_cache("spend:key:k1") == 3.5, "the local copy counts the cancelled spend"
     assert spend_cache.in_memory_cache.get_cache("spend:team:t1") is None, "an absent local copy is not seeded"
-
-
-@pytest.mark.asyncio
-async def test_the_update_cache_read_armed_before_accounting_rides_the_pipeline_of_the_reconcile_read():
-    from litellm.proxy.proxy_server import _read_update_cache_values, arm_update_cache_read
-
-    client = FakeClient(_ok_replies)
-    redis_cache = PostCallFakeRedisCache(client)
-    cache = DualCache()
-    cache.attach_redis_cache(redis_cache)
-    keys = ["user-1", "team_id:t1"]
-
-    with request_redis_batch_scope() as request:
-        await arm_update_cache_read(keys, cache=cache)
-        assert client.pipelines == []
-        await request.batch(redis_cache).mget(["spend:key:k1"])  # the spend reconcile read of the same request
-        values = await _read_update_cache_values(keys, None, cache=cache)
-
-    assert len(client.pipelines) == 1
-    assert client.pipelines[0].commands == [("MGET", "user-1", "team_id:t1"), ("MGET", "spend:key:k1")]
-    assert values == {"user-1": {"spend": 1.0}, "team_id:t1": {"spend": 1.0}}
-    assert redis_cache.alone == []
-    assert active_request_redis_batches() is None
-
-
-@pytest.mark.asyncio
-async def test_the_armed_update_cache_read_is_declared_under_the_auth_objects_family():
-    """The user, team and tag rows the accounting reads are auth objects, so the pipeline that carries
-    the armed read renders ``redis.pipeline auth_objects``, not a bare ``redis.pipeline``."""
-    from litellm.proxy.proxy_server import _read_update_cache_values, arm_update_cache_read
-
-    client = FakeClient(_ok_replies)
-    redis_cache = PostCallFakeRedisCache(client)
-    cache = DualCache()
-    cache.attach_redis_cache(redis_cache)
-    pipeline_targets: list[str | None] = []  # mutable-ok: filled by the recording hook
-
-    async def record(**kwargs: object) -> None:
-        pipeline_targets.append(current_service_target())
-
-    redis_cache.service_logger_obj.async_service_success_hook = record  # pyright: ignore[reportAttributeAccessIssue]  # fake, records the hook call
-
-    with request_redis_batch_scope():
-        await arm_update_cache_read(["user-1", "team_id:t1"], cache=cache)
-        await _read_update_cache_values(["user-1", "team_id:t1"], None, cache=cache)
-    await asyncio.gather(*(t for t in asyncio.all_tasks() if t is not asyncio.current_task()))
-
-    assert pipeline_targets == [AUTH_OBJECTS_TARGET]
-
-
-@pytest.mark.asyncio
-async def test_an_update_cache_read_armed_for_other_keys_is_ignored_and_the_read_happens_as_before():
-    from litellm.proxy.proxy_server import _read_update_cache_values, arm_update_cache_read
-
-    redis_cache = PostCallFakeRedisCache(FakeClient(_ok_replies))
-    redis_cache.store["team_id:t1"] = {"spend": 2.0}
-    cache = DualCache()
-    cache.attach_redis_cache(redis_cache)
-
-    with request_redis_batch_scope():
-        await arm_update_cache_read(["user-1"], cache=cache)
-        values = await _read_update_cache_values(["team_id:t1"], None, cache=cache)
-
-    assert values == {"team_id:t1": {"spend": 2.0}}
-    assert ("MGET", ("team_id:t1",)) in redis_cache.alone
-
-
-@pytest.mark.asyncio
-async def test_the_update_cache_read_sees_a_cached_spend_written_while_the_spend_was_persisted(monkeypatch):
-    from litellm.proxy import proxy_server
-    from litellm.proxy.hooks.proxy_track_cost_callback import _update_database_and_spend_counters
-
-    cached_user_spend = {"user-1": 1.0}
-
-    def replies(command: tuple[object, ...]) -> object:
-        if command[0] == "MGET":
-            return [
-                json.dumps({"spend": cached_user_spend[key]}) if key in cached_user_spend else b"0.5"
-                for key in command[1:]
-            ]
-        return _ok_replies(command)
-
-    client = FakeClient(replies)
-    redis_cache = PostCallFakeRedisCache(client)
-    spend_cache = DualCache()
-    spend_cache.attach_redis_cache(redis_cache)
-    user_cache = DualCache()
-    user_cache.attach_redis_cache(redis_cache)
-    monkeypatch.setattr(proxy_server, "spend_counter_cache", spend_cache)
-    monkeypatch.setattr(proxy_server, "user_api_key_cache", user_cache)
-
-    async def _read_on_the_request_pipeline_then_a_concurrent_callback_writes_the_user(**kwargs: object) -> bool:
-        request = active_request_redis_batches()
-        assert request is not None
-        await request.batch(redis_cache).mget(["key-object"])
-        cached_user_spend["user-1"] = 5.0
-        return True
-
-    proxy_logging_obj = MagicMock()
-    proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock(
-        side_effect=_read_on_the_request_pipeline_then_a_concurrent_callback_writes_the_user
-    )
-    reservation = {
-        "reserved_cost": 0.5,
-        "entries": [
-            {
-                "counter_key": "spend:key:k1",
-                "entity_type": "Key",
-                "entity_id": "k1",
-                "reserved_cost": 0.5,
-                "applied_adjustment": 0.0,
-            }
-        ],
-        "finalized": False,
-    }
-
-    with request_redis_batch_scope():
-        charged = await _update_database_and_spend_counters(
-            proxy_logging_obj=proxy_logging_obj,
-            increment_spend_counters=proxy_server.increment_spend_counters,
-            user_api_key="k1",
-            user_id="user-1",
-            end_user_id=None,
-            team_id=None,
-            org_id=None,
-            kwargs={},
-            completion_response=None,
-            start_time=datetime.datetime.now(),
-            end_time=datetime.datetime.now(),
-            response_cost=0.2,
-            budget_reservation=reservation,
-            update_cache_read_keys=("user-1",),
-        )
-        values = await proxy_server._read_update_cache_values(("user-1",), None)
-
-    assert charged is True
-    assert values == {"user-1": {"spend": 5.0}}, client.pipelines

tests/unit/integrations/otel/test_otel_v2_logger.py

@@ -1928,6 +1928,7 @@ def test_service_span_is_named_by_purpose_when_the_producer_declares_a_target():
         ("async_batch_get_cache", "redis.mget auth_objects", "redis.mget"),
         ("async_set_cache_pipeline_with_ttls", "redis.set auth_objects", "redis.set"),
         ("async_increment_pipeline", "redis.incr auth_objects", "redis.incr"),
+        ("async_increment_if_present", "redis.incr auth_objects", "redis.incr"),
         ("async_delete_cache", "redis.delete auth_objects", "redis.delete"),
         ("async_scan_iter", "redis.scan auth_objects", "redis.scan"),
         ("request_redis_batch", "redis.pipeline auth_objects", "redis.pipeline"),

tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py

@@ -10,6 +10,7 @@ from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
 from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
 
 
 class _ProbedCountTokensHandler(BedrockCountTokensHandler):
@@ -56,6 +57,34 @@ async def test_handle_count_tokens_request_signs_off_the_event_loop(monkeypatch)
     assert probe.served_during_refresh is True
 
 
+def _count_tokens_in_us_west_2(request: httpx.Request) -> httpx.Response:
+    if request.url.host != "bedrock-runtime.us-west-2.amazonaws.com":
+        return httpx.Response(404, request=request)
+    return httpx.Response(200, json={"inputTokens": 7}, request=request)
+
+
+@pytest.mark.asyncio
+async def test_handle_count_tokens_request_looks_up_its_region_off_the_event_loop(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
+    monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATESTACCESSKEY")
+    monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
+    monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+    vault: Final = host_in_loop_probing_vault(monkeypatch, "AWS_REGION_NAME", "AWS_BEDROCK_RUNTIME_ENDPOINT")
+
+    result = await BedrockCountTokensHandler().handle_count_tokens_request(
+        request_data={"model": "us.anthropic.claude-sonnet-5-5", "messages": [{"role": "user", "content": "hi"}]},
+        litellm_params={},
+        resolved_model="us.anthropic.claude-sonnet-5-5",
+        client=AsyncHTTPHandler(transport=httpx.MockTransport(_count_tokens_in_us_west_2)),
+    )
+
+    assert vault.read.is_set()
+    assert not vault.stalled_the_loop.is_set()
+    assert result == {"input_tokens": 7}
+
+
 class TestBedrockCountTokensEndpoint:
     def _make_handler(self) -> BedrockCountTokensConfig:
         return BedrockCountTokensConfig()

tests/unit/llms/bedrock/count_tokens/test_bedrock_mantle_count_tokens_handler.py

@@ -8,6 +8,7 @@ import pytest
 from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
 
 MANTLE_COUNT_URL: Final = "https://bedrock-mantle.us-east-1.api.aws/anthropic/v1/messages/count_tokens"
 LITELLM_PARAMS: Final = {
@@ -107,3 +108,22 @@ async def test_non_200_answers_raise_bedrock_error_with_mantle_status() -> None:
 
     assert raised.value.status_code == 404
     assert "does not exist" in raised.value.message
+
+
+@pytest.mark.asyncio
+async def test_region_and_endpoint_lookups_leave_the_event_loop_free(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
+    vault: Final = host_in_loop_probing_vault(monkeypatch, "AWS_REGION_NAME", "BEDROCK_MANTLE_API_BASE")
+    mantle: Final = _MantleEndpoint(200, {"input_tokens": 2177})
+
+    result = await BedrockMantleCountTokensHandler().handle_count_tokens_request(
+        request_data={**REQUEST, "model": "global.anthropic.claude-opus-5-5"},
+        litellm_params={key: value for key, value in LITELLM_PARAMS.items() if key != "aws_region_name"},
+        resolved_model="anthropic.claude-opus-5-5",
+        client=_client(mantle),
+    )
+
+    assert vault.read.is_set()
+    assert not vault.stalled_the_loop.is_set()
+    assert result == {"input_tokens": 2177}
+    assert str(mantle.only_request().url) == MANTLE_COUNT_URL

tests/unit/llms/bedrock/event_loop_probe.py

@@ -1,4 +1,4 @@
-"""Refreshable credentials whose refresh only completes while the event loop keeps serving."""
+"""Credential refreshes and Vault reads that only complete while the event loop keeps serving."""
 
 from __future__ import annotations
 
@@ -8,10 +8,15 @@ import time
 from datetime import datetime, timedelta, timezone
 from typing import Final
 
+import pytest
 from botocore.credentials import RefreshableCredentials
 
+import litellm
+from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
+
 REFRESH_RELEASE_TIMEOUT_SECONDS: Final = 2.0
 REFRESH_START_TIMEOUT_SECONDS: Final = 10.0
+LOOP_SERVE_TIMEOUT_SECONDS: Final = 2.0
 
 
 class EventLoopProbe:
@@ -55,3 +60,34 @@ class EventLoopProbe:
                 raise TimeoutError("signing finished without ever starting a credential refresh")
             await asyncio.sleep(0.005)
         self.loop_served.set()
+
+
+class LoopProbingVault:
+    """A Vault holding none of its hosted secrets, whose read waits for the event loop to run a callback.
+
+    A read made on the loop thread can never see that callback, so `stalled_the_loop` is set only then.
+    """
+
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+        self._loop: Final = loop
+        self.read: Final = threading.Event()
+        self.stalled_the_loop: Final = threading.Event()
+
+    def sync_read_secret(self, secret_name: str) -> None:
+        self.read.set()
+        served: Final = threading.Event()
+        self._loop.call_soon_threadsafe(served.set)
+        if not served.wait(timeout=LOOP_SERVE_TIMEOUT_SECONDS):
+            self.stalled_the_loop.set()
+
+
+def host_in_loop_probing_vault(monkeypatch: pytest.MonkeyPatch, *secret_names: str) -> LoopProbingVault:
+    vault: Final = LoopProbingVault(asyncio.get_running_loop())
+    monkeypatch.setattr(litellm, "secret_manager_client", vault)
+    monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.HASHICORP_VAULT)
+    monkeypatch.setattr(
+        litellm,
+        "_key_management_settings",
+        KeyManagementSettings(access_mode="read_only", hosted_keys=list(secret_names)),
+    )
+    return vault

tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py

@@ -18,6 +18,7 @@ from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.llms.bedrock.realtime.handler import BedrockRealtime
 from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
 from litellm.realtime_api import main as realtime_main
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
 
 
 @pytest.fixture(autouse=True)
@@ -625,6 +626,27 @@ class TestBedrockRealtimeSessionLifecycle:
         assert first_event["session"]["id"] == "trace-nova-sonic"
         assert first_event["session"]["model"] == "amazon.nova-sonic-v1:0"
 
+    @pytest.mark.asyncio
+    async def test_region_lookup_on_connect_leaves_the_event_loop_free(
+        self, stub_aws_sdk_client, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+        vault: Final = host_in_loop_probing_vault(monkeypatch, "AWS_REGION_NAME")
+        websocket: Final = RealtimeClientWS()
+
+        _ = await BedrockRealtime().async_realtime(
+            model="amazon.nova-2-sonic-v1:0",
+            websocket=websocket,
+            logging_obj=FakeLogging(),
+            aws_access_key_id="k",
+            aws_secret_access_key="s",
+        )
+
+        assert vault.read.is_set()
+        assert not vault.stalled_the_loop.is_set()
+        assert stub_aws_sdk_client["config_kwargs"]["region"] == "us-west-2"
+        assert stub_aws_sdk_client["config_kwargs"]["endpoint_uri"] == "https://bedrock-runtime.us-west-2.amazonaws.com"
+
     @pytest.mark.asyncio
     async def test_session_update_is_acked_with_session_updated(self, stub_aws_models):
         handler = BedrockRealtime()

tests/unit/llms/bedrock/test_base_aws_llm.py

@@ -30,7 +30,7 @@ from litellm.llms.bedrock.base_aws_llm import (
     BaseAWSLLM,
     Boto3CredentialsInfo,
     run_aws_signing,
-    sign_request_off_loop_if_aws,
+    run_off_loop_if_aws,
 )
 from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
 from litellm.llms.bedrock.common_utils import BedrockModelInfo
@@ -3443,7 +3443,7 @@ class TestGetRequestHeadersResign:
 
 
 @pytest.mark.asyncio
-async def test_sign_request_off_loop_if_aws_keeps_the_loop_serving_while_credentials_refresh():
+async def test_run_off_loop_if_aws_keeps_the_loop_serving_while_credentials_refresh() -> None:
     """Regression for issue #40165: an AWS provider's signing (and the botocore credential refresh
     inside it) must run off the event loop, so other requests keep being served meanwhile."""
     probe = EventLoopProbe()
@@ -3456,7 +3456,7 @@ async def test_sign_request_off_loop_if_aws_keeps_the_loop_serving_while_credent
         return dict(request.headers)
 
     release = asyncio.create_task(probe.release_refresh_from_the_loop())
-    signed = await sign_request_off_loop_if_aws(BaseAWSLLM(), sign, headers={"Content-Type": "application/json"})
+    signed = await run_off_loop_if_aws(BaseAWSLLM(), sign, headers={"Content-Type": "application/json"})
     await release
 
     assert "Authorization" in signed

tests/unit/llms/bedrock/test_claude_platform_provider.py

@@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError
 
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.types.router import GenericLiteLLMParams
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
 
 WORKSPACE_ALIASES: Final = ("workspace_id", "aws_workspace_id", "anthropic_workspace_id", "anthropic-workspace-id")
 
@@ -683,6 +684,32 @@ async def test_anthropic_messages_claude_platform_sends_exact_body_through_stric
     _assert_gateway_accepted(requests[0], auth_kwargs)
 
 
+@pytest.mark.asyncio
+async def test_anthropic_messages_claude_platform_workspace_and_region_lookups_leave_the_event_loop_free(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    import litellm
+
+    monkeypatch.setenv("ANTHROPIC_AWS_WORKSPACE_ID", "wrkspc_test")
+    monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+    vault: Final = host_in_loop_probing_vault(monkeypatch, "ANTHROPIC_AWS_WORKSPACE_ID", "AWS_REGION_NAME")
+    sigv4_without_region: Final = {key: value for key, value in SIGV4_KWARGS.items() if key != "aws_region_name"}
+    requests: list[dict] = []
+
+    response = await litellm.anthropic_messages(
+        model="bedrock/claude_platform/claude-sonnet-5-5",
+        messages=[{"role": "user", "content": "hello"}],
+        max_tokens=10,
+        client=_async_gateway_client(requests),
+        **sigv4_without_region,
+    )
+
+    assert vault.read.is_set()
+    assert not vault.stalled_the_loop.is_set()
+    assert response["content"][0]["text"] == "ok"
+    _assert_gateway_accepted(requests[0], sigv4_without_region)
+
+
 def test_chat_completion_claude_platform_drops_context_management_and_gateway_accepts():
     import litellm
 

tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py

@@ -20,7 +20,7 @@ from botocore.awsrequest import AWSRequest
 import litellm
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig
-from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws
+from litellm.llms.bedrock.base_aws_llm import run_off_loop_if_aws
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 from litellm.types.utils import LlmProviders
 from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
@@ -1234,9 +1234,7 @@ async def test_mantle_signing_runs_off_the_event_loop():
         return dict(request.headers)
 
     release = asyncio.create_task(probe.release_refresh_from_the_loop())
-    signed = await sign_request_off_loop_if_aws(
-        BedrockMantleChatConfig(), sign, headers={"Content-Type": "application/json"}
-    )
+    signed = await run_off_loop_if_aws(BedrockMantleChatConfig(), sign, headers={"Content-Type": "application/json"})
     await release
 
     assert "Authorization" in signed

tests/unit/llms/custom_httpx/test_llm_http_handler.py

@@ -5,13 +5,14 @@ import json
 import logging
 import threading
 import time
-from collections.abc import Mapping
+from collections.abc import Awaitable, Callable, Mapping
 from types import MappingProxyType
 from typing import Final
 from unittest.mock import AsyncMock, Mock, patch
 
 import httpx
 import pytest
+import respx
 from botocore.credentials import RefreshableCredentials
 
 import litellm
@@ -66,6 +67,7 @@ from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIRe
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import ImageObject, ImageResponse, ModelResponse, TranscriptionResponse
 from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
 
 _ACTIVE_KEY = "_code_interpreter_interception_active"
 _SANDBOX_KEY = "_code_interpreter_interception_sandbox_key"
@@ -1245,6 +1247,155 @@ async def test_async_anthropic_messages_handler_signs_bedrock_off_the_event_loop
     assert probe.served_during_refresh is True
 
 
+_BEDROCK_MESSAGE: Final = {
+    "id": "msg_1",
+    "type": "message",
+    "role": "assistant",
+    "model": "claude-sonnet-5-5",
+    "content": [{"type": "text", "text": "hi"}],
+    "stop_reason": "end_turn",
+    "usage": {"input_tokens": 1, "output_tokens": 1},
+}
+_BEDROCK_RESPONSE: Final = {
+    "id": "resp_1",
+    "object": "response",
+    "created_at": 1,
+    "status": "completed",
+    "model": "openai.gpt-5.6-sol",
+    "output": [],
+    "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
+}
+_EMPTY_S3_LISTING: Final = (
+    b'<?xml version="1.0" encoding="UTF-8"?>'
+    b'<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
+    b"<Name>batch-inputs</Name><KeyCount>0</KeyCount><IsTruncated>false</IsTruncated></ListBucketResult>"
+)
+_BATCH_LINE: Final = json.dumps(
+    {
+        "custom_id": "1",
+        "method": "POST",
+        "url": "/v1/chat/completions",
+        "body": {"model": "us.anthropic.claude-sonnet-5-5", "messages": [{"role": "user", "content": "hi"}]},
+    }
+).encode()
+_BATCH_FILE_ID: Final = "s3://batch-inputs/litellm-bedrock-files/batch.jsonl"
+_BEDROCK_ENTRY_POINTS: Final[tuple[tuple[str, Callable[[], Awaitable[object]], httpx.Response], ...]] = (
+    (
+        "anthropic_messages",
+        lambda: litellm.anthropic.messages.acreate(
+            model="bedrock/invoke/us.anthropic.claude-sonnet-5-5",
+            messages=[{"role": "user", "content": "hi"}],
+            max_tokens=16,
+        ),
+        httpx.Response(200, json=_BEDROCK_MESSAGE),
+    ),
+    (
+        "responses",
+        lambda: litellm.aresponses(model="bedrock/us.openai.gpt-5.6-sol", input="hi"),
+        httpx.Response(200, json=_BEDROCK_RESPONSE),
+    ),
+    (
+        "compact_responses",
+        lambda: litellm.acompact_responses(model="bedrock/us.openai.gpt-5.6-sol", input="hi"),
+        httpx.Response(200, json=_BEDROCK_RESPONSE),
+    ),
+    (
+        "search",
+        lambda: litellm.asearch(
+            query="hi",
+            search_provider="agentcore",
+            api_base="https://gw.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp",
+        ),
+        httpx.Response(200, json={"jsonrpc": "2.0", "id": 1, "result": {"content": []}}),
+    ),
+    (
+        "create_file",
+        lambda: litellm.acreate_file(
+            file=("batch.jsonl", _BATCH_LINE, "application/jsonl"),
+            purpose="batch",
+            custom_llm_provider="bedrock",
+            model="bedrock/us.anthropic.claude-sonnet-5-5",
+        ),
+        httpx.Response(200),
+    ),
+    (
+        "retrieve_file",
+        lambda: litellm.afile_retrieve(file_id=_BATCH_FILE_ID, custom_llm_provider="bedrock"),
+        httpx.Response(200, content=_BATCH_LINE),
+    ),
+    (
+        "delete_file",
+        lambda: litellm.afile_delete(file_id=_BATCH_FILE_ID, custom_llm_provider="bedrock"),
+        httpx.Response(204),
+    ),
+    (
+        "list_files",
+        lambda: litellm.afile_list(custom_llm_provider="bedrock"),
+        httpx.Response(200, content=_EMPTY_S3_LISTING),
+    ),
+    (
+        "file_content",
+        lambda: litellm.afile_content(file_id=_BATCH_FILE_ID, custom_llm_provider="bedrock"),
+        httpx.Response(200, content=_BATCH_LINE),
+    ),
+    (
+        "create_batch",
+        lambda: litellm.acreate_batch(
+            completion_window="24h",
+            endpoint="/v1/chat/completions",
+            input_file_id=_BATCH_FILE_ID,
+            custom_llm_provider="bedrock",
+            model="bedrock/us.anthropic.claude-sonnet-5-5",
+            aws_batch_role_arn="arn:aws:iam::123456789012:role/batch",
+            s3_output_bucket_name="batch-outputs",
+        ),
+        httpx.Response(200, json={"jobArn": "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/batch"}),
+    ),
+    (
+        "vector_store_search",
+        lambda: litellm.vector_stores.asearch(vector_store_id="KBID12345", query="hi", custom_llm_provider="bedrock"),
+        httpx.Response(200, json={"retrievalResults": []}),
+    ),
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("call", "upstream_answer"),
+    [pytest.param(call, answer, id=name) for name, call, answer in _BEDROCK_ENTRY_POINTS],
+)
+async def test_bedrock_secret_lookups_leave_the_event_loop_free(
+    call: Callable[[], Awaitable[object]],
+    upstream_answer: httpx.Response,
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATESTTESTTESTTEST")
+    monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
+    monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+    monkeypatch.setenv("AWS_S3_BUCKET_NAME", "batch-inputs")
+    monkeypatch.setenv("AWS_S3_OUTPUT_BUCKET_NAME", "batch-outputs")
+    vault: Final = host_in_loop_probing_vault(
+        monkeypatch,
+        "AWS_REGION_NAME",
+        "AWS_BEDROCK_RUNTIME_ENDPOINT",
+        "AWS_BEARER_TOKEN_BEDROCK",
+        "AWS_S3_ENCRYPTION_KEY_ID",
+        "AWS_S3_BUCKET_OWNER",
+        "AGENTCORE_MCP_PROTOCOL_VERSION",
+        "AGENTCORE_SEARCH_TOOL_NAME",
+        "AGENTCORE_GATEWAY_TOKEN",
+    )
+    upstream: Final = respx_mock.route().mock(return_value=upstream_answer)
+
+    _ = await call()
+
+    assert vault.read.is_set()
+    assert not vault.stalled_the_loop.is_set()
+    assert ".us-west-2." in upstream.calls.last.request.url.host
+
+
 @pytest.mark.asyncio
 async def test_async_anthropic_messages_handler_passes_litellm_metadata():
     """Ensure litellm_metadata from kwargs is forwarded via update_from_kwargs.

tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py

@@ -35,6 +35,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
 )
 from litellm.types.utils import CallTypes, ModelResponse
 from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
 
 
 @pytest.fixture
@@ -8063,3 +8064,44 @@ async def test_during_call_masking_rewrites_only_pii_block_in_content_list(
     ]
     assert messages[1]["content"] == "Hello, how can I help you today?"
     assert messages[2]["content"] == "who is the president of the united states?"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("guardrail_config", "upstream_answer"),
+    [
+        pytest.param(
+            {"guardrailIdentifier": "wf0hkdb5x07f", "guardrailVersion": "DRAFT"},
+            {"action": "NONE", "outputs": [], "assessments": []},
+            id="apply_guardrail",
+        ),
+        pytest.param(
+            {"checks": {"contentFilter": {"categories": [{"category": "VIOLENCE"}]}}, "content_filter_threshold": 0.5},
+            {"results": {"contentFilter": {"results": [{"category": "VIOLENCE", "severityScore": 0.1}]}}},
+            id="invoke_guardrail_checks",
+        ),
+    ],
+)
+async def test_guardrail_secret_lookups_leave_the_event_loop_free(
+    guardrail_config: dict[str, object],
+    upstream_answer: dict[str, object],
+    respx_mock: respx.MockRouter,
+    httpx_transport: None,
+    aws_test_credentials: None,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
+    monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+    vault: Final = host_in_loop_probing_vault(monkeypatch, "AWS_BEARER_TOKEN_BEDROCK", "AWS_REGION_NAME")
+    guardrail: Final = BedrockGuardrail(**guardrail_config)
+    upstream: Final = respx_mock.route().mock(return_value=httpx.Response(200, json=upstream_answer))
+
+    _ = await guardrail.make_bedrock_api_request(
+        source="INPUT",
+        messages=[{"role": "user", "content": "hello"}],
+        request_data={"messages": []},
+    )
+
+    assert vault.read.is_set()
+    assert not vault.stalled_the_loop.is_set()
+    assert upstream.calls.last.request.url.host == "bedrock-runtime.us-west-2.amazonaws.com"

tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py

@@ -35,6 +35,7 @@ from litellm.proxy.utils import ProxyLogging
 from litellm.passthrough.main import raise_for_provider_status
 from litellm.llms.bedrock.common_utils import BedrockError
 from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
 from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
 from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     BaseOpenAIPassThroughHandler,
@@ -2519,6 +2520,48 @@ class TestBedrockAgentRuntimePassthroughVirtualKeyLeak:
         assert upstream["x-request-id"] == "trace-2"
 
 
+@pytest.mark.asyncio
+async def test_bedrock_agent_runtime_region_lookup_leaves_the_event_loop_free(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr("litellm.proxy.proxy_server.master_key", SHARED_MASTER_KEY)
+    monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    litellm.in_memory_llm_clients_cache.flush_cache()
+    for ambient in TestBedrockAgentRuntimePassthroughVirtualKeyLeak.AMBIENT_AWS_ENV:
+        monkeypatch.delenv(ambient, raising=False)
+    monkeypatch.setenv("AWS_ACCESS_KEY_ID", "ak")
+    monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "sk")
+    monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+    vault: Final = host_in_loop_probing_vault(monkeypatch, "AWS_REGION_NAME")
+
+    async def receive() -> dict[str, object]:
+        return {"type": "http.request", "body": b'{"retrievalQuery": {"text": "hi"}}', "more_body": False}
+
+    request: Final = Request(
+        {
+            "type": "http",
+            "method": "POST",
+            "path": "/bedrock/knowledgebases/KB1234567/retrieve",
+            "headers": [(b"content-type", b"application/json")],
+            "query_string": b"",
+        },
+        receive=receive,
+    )
+    with respx.mock(assert_all_called=True) as upstream:
+        retrieve: Final = upstream.post(
+            "https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases/KB1234567/retrieve"
+        ).mock(return_value=httpx.Response(200, json={"retrievalResults": []}))
+        _ = await bedrock_proxy_route(
+            endpoint="knowledgebases/KB1234567/retrieve",
+            request=request,
+            fastapi_response=Response(),
+            user_api_key_dict=UserAPIKeyAuth(api_key="sk-litellm-caller"),
+        )
+
+    assert vault.read.is_set()
+    assert not vault.stalled_the_loop.is_set()
+    assert retrieve.calls.last.request.headers["authorization"].startswith("AWS4-HMAC-SHA256")
+
+
 class TestLLMPassthroughFactoryProxyRoute:
     @pytest.mark.asyncio
     async def test_llm_passthrough_factory_proxy_route_success(self):

tests/unit/proxy/proxy_server/test_spend_counters.py

@@ -24,15 +24,31 @@ from __future__ import annotations
 import asyncio
 from collections.abc import Awaitable, Callable
 from datetime import datetime
-from typing import Final, cast
-from unittest.mock import AsyncMock, MagicMock
+from typing import Final, Literal, cast
+from unittest.mock import AsyncMock, MagicMock, call
 
 import pytest
+from fakeredis import FakeAsyncRedis, FakeServer
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import StatusCode
 
+import litellm
 import litellm.proxy.proxy_server as ps
+from litellm._service_logger import ServiceLogging
+from litellm._internal_context import current_service_target
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker
+from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY
+from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
+from litellm.models.end_user import LiteLLM_EndUserTable
+from litellm.models.tag import LiteLLM_TagTable
 from litellm.models.team import LiteLLM_TeamTable, LiteLLM_TeamTableCachedObj
+from litellm.models.user import LiteLLM_UserTable
 from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_local
-from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
+from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_user_cache_key, tag_cache_key
 from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team
 
 from .conftest import normalize
@@ -1559,31 +1575,6 @@ async def test_invalidate_spend_counter_swallows_redis_failure_no_raise(monkeypa
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.asyncio
-async def test_update_cache_no_cached_entities_schedules_pipeline_flush(monkeypatch):
-    fake_user_cache = _make_user_api_key_cache(get_value=None)
-    monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache)
-
-    await ps.update_cache(
-        token=None,
-        user_id="u1",
-        end_user_id="eu1",
-        team_id="t1",
-        response_cost=1.0,
-        parent_otel_span=None,
-        tags=["x"],
-    )
-
-    assert fake_user_cache.async_get_cache.await_count == 0
-    fake_user_cache.async_batch_get_cache.assert_awaited_once()
-    assert fake_user_cache.async_batch_get_cache.await_args.kwargs["keys"] == [
-        "u1",
-        f"{ps.litellm_proxy_admin_name}:spend",
-        "end_user_id:eu1",
-        "tag:x",
-    ]
-
-
 @pytest.mark.asyncio
 async def test_update_cache_user_cache_failure_invalid_state_is_swallowed(monkeypatch):
     """An inner _update_user_cache raising must not propagate — update_cache
@@ -1671,3 +1662,354 @@ async def test_update_cache_never_restores_a_team_a_management_write_replaced(
 
     seen_by_auth: Final = cache.get_cache(key=f"team_id:{team_id}", model_type=LiteLLM_TeamTableCachedObj)
     assert (seen_by_auth.blocked if seen_by_auth is not None else None) == expected_blocked
+
+
+_CachedSpendObject = LiteLLM_UserTable | LiteLLM_EndUserTable | LiteLLM_TeamTableCachedObj | LiteLLM_TagTable
+
+
+def _cached_spend_object(
+    kind: Literal["user", "end_user", "team", "tag"], models: list[str]
+) -> tuple[str, _CachedSpendObject]:
+    match kind:
+        case "user":
+            return "race-user", LiteLLM_UserTable(user_id="race-user", models=models)
+        case "end_user":
+            return end_user_cache_key("race-end-user"), LiteLLM_EndUserTable(
+                user_id="race-end-user", blocked=False, models=models
+            )
+        case "team":
+            return "team_id:race-team", LiteLLM_TeamTableCachedObj(team_id="race-team", models=models)
+        case "tag":
+            return tag_cache_key("race-tag"), LiteLLM_TagTable(tag_name="race-tag", models=models)
+
+
+class _RunsOnKeyRead(InMemoryCache):
+    def __init__(self, on_read: Callable[[], Awaitable[None]]) -> None:
+        super().__init__()
+        self._on_read: Final = on_read
+
+    async def async_get_cache(self, key: str, **kwargs: object) -> object:
+        await self._on_read()
+        return await super().async_get_cache(key, **kwargs)
+
+
+def _cache_that_runs_on_key_read(on_read: Callable[[], Awaitable[None]]) -> UserApiKeyCache:
+    return UserApiKeyCache(in_memory_cache=InMemoryCache(), key_object_in_memory_cache=_RunsOnKeyRead(on_read))
+
+
+@pytest.fixture
+def proxy_max_budget(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(litellm, "max_budget", 100.0)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["user", "end_user", "team", "tag"])
+@pytest.mark.parametrize(
+    ("management_write", "expected"),
+    [("none", (["gpt-a"], 0.25)), ("evict", None), ("refresh", (["gpt-b"], 0.25))],
+)
+async def test_update_cache_adds_spend_to_the_entry_a_management_write_left_mid_flight(
+    kind: Literal["user", "end_user", "team", "tag"],
+    management_write: Literal["none", "evict", "refresh"],
+    expected: tuple[list[str], float] | None,
+) -> None:
+    key, seeded = _cached_spend_object(kind, models=["gpt-a"])
+    _, refreshed = _cached_spend_object(kind, models=["gpt-b"])
+
+    async def _write() -> None:
+        if management_write == "none":
+            return
+        evict_local(cache_keys=(key,), user_api_key_cache=cache)
+        if management_write == "refresh":
+            cache.set_cache(key=key, value=refreshed, local_only=True)
+
+    cache: Final = _cache_that_runs_on_key_read(on_read=_write)
+    cache.set_cache(key=key, value=seeded, local_only=True)
+
+    await ps.update_cache(
+        token="sk-race",
+        user_id="race-user",
+        end_user_id="race-end-user",
+        team_id="race-team",
+        response_cost=0.25,
+        parent_otel_span=None,
+        tags=["race-tag"],
+        cache=cache,
+    )
+    await _drain_spend_cache_writes()
+
+    stored: Final = cache.get_cache(key=key, local_only=True, model_type=type(seeded))
+    assert ((stored.models, stored.spend) if stored is not None else None) == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+@pytest.mark.parametrize("user", ["cached_on_this_worker", "not_cached_on_this_worker", "none"])
+async def test_update_cache_adds_spend_to_the_global_proxy_spend_for_every_request(
+    user: Literal["cached_on_this_worker", "not_cached_on_this_worker", "none"],
+) -> None:
+    cache: Final = UserApiKeyCache()
+    cache.set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0)
+    if user == "cached_on_this_worker":
+        cache.set_cache(key="spender", value=LiteLLM_UserTable(user_id="spender", spend=1.0))
+
+    await ps.update_cache(
+        token=None,
+        user_id=None if user == "none" else "spender",
+        end_user_id=None,
+        team_id=None,
+        response_cost=0.25,
+        parent_otel_span=None,
+        cache=cache,
+    )
+    await _drain_spend_cache_writes()
+
+    assert cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY) == 10.25
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+async def test_update_cache_never_brings_back_a_global_proxy_spend_a_budget_reset_cleared() -> None:
+    async def _budget_reset() -> None:
+        await cache.async_delete_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY)
+
+    cache: Final = _cache_that_runs_on_key_read(on_read=_budget_reset)
+    cache.set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0)
+
+    await ps.update_cache(
+        token="sk-spender",
+        user_id="spender",
+        end_user_id=None,
+        team_id=None,
+        response_cost=0.25,
+        parent_otel_span=None,
+        cache=cache,
+    )
+    await _drain_spend_cache_writes()
+
+    assert cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY) is None
+
+
+class _FakeRedisRunningIncrementIfPresent(FakeAsyncRedis):
+    def __init__(self, server: FakeServer, after_increment: Callable[[], object] = lambda: None) -> None:
+        super().__init__(server=server)
+        self._after_increment: Final = after_increment
+
+    async def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> bytes | None:
+        key, increment = keys_and_args
+        if not await self.exists(key):
+            return None
+        total: Final = await self.incrbyfloat(key, increment)
+        self._after_increment()
+        return str(total).encode()
+
+
+class _RedisCacheOnClient(RedisCache):
+    def __init__(self, client: FakeAsyncRedis) -> None:
+        self._injected_client: Final = client
+        self.namespace = None
+        self._circuit_breaker = RedisCircuitBreaker(failure_threshold=5, recovery_timeout=30)
+        self.service_logger_obj = ServiceLogging()
+        self.default_ttl = None
+
+    def init_async_client(self) -> FakeAsyncRedis:
+        return self._injected_client
+
+
+class _FakeRedisHoldingOneReply(FakeAsyncRedis):
+    def __init__(self, server: FakeServer, held_increment: float, applied: asyncio.Event, released: asyncio.Event) -> None:
+        super().__init__(server=server)
+        self._held_increment: Final = held_increment
+        self._applied: Final = applied
+        self._released: Final = released
+
+    async def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> bytes | None:
+        key, increment = keys_and_args
+        total: Final = await self.incrbyfloat(key, increment)
+        if float(increment) == self._held_increment:
+            self._applied.set()
+            await self._released.wait()
+        return str(total).encode()
+
+
+_SharedGlobalSpendOutcome = Literal["peers_added_spend", "reset_elsewhere", "reset_while_redis_answers", "redis_down"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+@pytest.mark.parametrize(
+    ("outcome", "shared_total", "expected"),
+    [
+        ("peers_added_spend", 11.75, 12.0),
+        ("reset_elsewhere", None, 10.25),
+        ("reset_while_redis_answers", 9.75, None),
+        ("redis_down", 10.0, 10.25),
+    ],
+)
+async def test_update_cache_moves_this_workers_global_proxy_spend_to_the_shared_total(
+    outcome: _SharedGlobalSpendOutcome, shared_total: float | None, expected: float | None
+) -> None:
+    def _reset_this_worker() -> None:
+        if outcome == "reset_while_redis_answers":
+            cache.in_memory_cache.delete_cache(GLOBAL_PROXY_SPEND_CACHE_KEY)
+
+    server: Final = FakeServer()
+    redis: Final = _FakeRedisRunningIncrementIfPresent(server, after_increment=_reset_this_worker)
+    if shared_total is not None:
+        await redis.set(GLOBAL_PROXY_SPEND_CACHE_KEY, shared_total)
+    server.connected = outcome != "redis_down"
+    cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnClient(redis))
+    cache.set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0, local_only=True)
+
+    await ps.update_cache(
+        token=None,
+        user_id=None,
+        end_user_id=None,
+        team_id=None,
+        response_cost=0.25,
+        parent_otel_span=None,
+        cache=cache,
+    )
+
+    assert cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, local_only=True) == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+async def test_update_cache_keeps_the_newest_global_proxy_spend_when_redis_replies_out_of_order() -> None:
+    held_increment_applied: Final = asyncio.Event()
+    held_reply_released: Final = asyncio.Event()
+    server: Final = FakeServer()
+    redis: Final = _FakeRedisHoldingOneReply(
+        server, held_increment=0.25, applied=held_increment_applied, released=held_reply_released
+    )
+    await redis.set(GLOBAL_PROXY_SPEND_CACHE_KEY, 10.0)
+    cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnClient(redis))
+    cache.set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0, local_only=True)
+
+    async def _request(response_cost: float) -> None:
+        await ps.update_cache(
+            token=None,
+            user_id=None,
+            end_user_id=None,
+            team_id=None,
+            response_cost=response_cost,
+            parent_otel_span=None,
+            cache=cache,
+        )
+
+    earlier: Final = asyncio.create_task(_request(0.25))
+    await held_increment_applied.wait()
+    await _request(0.5)
+    held_reply_released.set()
+    await earlier
+
+    assert cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, local_only=True) == 10.75
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+async def test_update_cache_adds_the_global_spend_scalar_in_redis_as_spend_counters() -> None:
+    target_during_increment: Final = MagicMock()
+    redis: Final = _FakeRedisRunningIncrementIfPresent(
+        FakeServer(), after_increment=lambda: target_during_increment(current_service_target())
+    )
+    await redis.set(GLOBAL_PROXY_SPEND_CACHE_KEY, 10.0)
+    cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnClient(redis))
+
+    await ps.update_cache(
+        token=None,
+        user_id="u1",
+        end_user_id=None,
+        team_id=None,
+        response_cost=2.0,
+        parent_otel_span=None,
+        cache=cache,
+    )
+
+    assert target_during_increment.call_args_list == [call("spend_counters")]
+    assert float(await redis.get(GLOBAL_PROXY_SPEND_CACHE_KEY)) == 12.0
+    assert current_service_target() is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+@pytest.mark.parametrize(("redis_up", "expected_status"), [(True, StatusCode.OK), (False, StatusCode.ERROR)])
+async def test_update_cache_traces_the_global_spend_redis_call_under_the_request_span(
+    monkeypatch: pytest.MonkeyPatch, redis_up: bool, expected_status: StatusCode
+) -> None:
+    exporter: Final = InMemorySpanExporter()
+    provider: Final = TracerProvider()
+    provider.add_span_processor(SimpleSpanProcessor(exporter))
+    monkeypatch.setattr(
+        litellm,
+        "service_callback",
+        [OpenTelemetry(config=OpenTelemetryConfig(exporter=exporter), tracer_provider=provider)],
+    )
+    server: Final = FakeServer()
+    redis: Final = _FakeRedisRunningIncrementIfPresent(server)
+    await redis.set(GLOBAL_PROXY_SPEND_CACHE_KEY, 10.0)
+    server.connected = redis_up
+    cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnClient(redis))
+    request_span: Final = provider.get_tracer("test").start_span("request")
+
+    await ps.update_cache(
+        token=None,
+        user_id=None,
+        end_user_id=None,
+        team_id=None,
+        response_cost=2.0,
+        parent_otel_span=request_span,
+        cache=cache,
+    )
+    await _drain_spend_cache_writes()
+    request_span.end()
+
+    redis_spans: Final = tuple(
+        (span.attributes.get("call_type"), span.status.status_code, span.parent.span_id)
+        for span in exporter.get_finished_spans()
+        if span.name == "redis" and span.attributes is not None and span.parent is not None
+    )
+    assert redis_spans == (
+        ("async_increment_if_present", expected_status, request_span.get_span_context().span_id),
+    )
+
+
+class _UnwritableKeyUserApiKeyCache(UserApiKeyCache):
+    def __init__(self, unwritable_key: str) -> None:
+        super().__init__(in_memory_cache=InMemoryCache())
+        self._unwritable_key: Final = unwritable_key
+
+    def set_cache(self, key: str | None, value: object, local_only: bool = False, **kwargs: object):
+        if key == self._unwritable_key:
+            raise RuntimeError("cache write failed")
+        return super().set_cache(key, value, local_only=local_only, **kwargs)
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+async def test_update_cache_still_adds_spend_to_the_others_when_one_cached_object_fails() -> None:
+    cache: Final = _UnwritableKeyUserApiKeyCache(unwritable_key="broken-user")
+    cache.in_memory_cache.set_cache(
+        key="broken-user",
+        value=CacheCodec.serialize(LiteLLM_UserTable(user_id="broken-user", spend=1.0), model_type=LiteLLM_UserTable),
+    )
+    cache.set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0)
+    cache.set_cache(key="team_id:still-billed", value=LiteLLM_TeamTableCachedObj(team_id="still-billed", spend=1.0))
+
+    await ps.update_cache(
+        token=None,
+        user_id="broken-user",
+        end_user_id=None,
+        team_id="still-billed",
+        response_cost=0.25,
+        parent_otel_span=None,
+        cache=cache,
+    )
+    await _drain_spend_cache_writes()
+
+    team: Final = cache.get_cache(key="team_id:still-billed", model_type=LiteLLM_TeamTableCachedObj)
+    assert (team.spend if team is not None else None, cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY)) == (
+        1.25,
+        10.25,
+    )

tests/unit/proxy/spend_tracking/test_spend_counter_batch.py

@@ -737,22 +737,3 @@ async def test_a_resized_reservation_is_carried_at_its_resized_cost_to_the_count
     assert reservation["reserved_cost"] == pytest.approx(0.1)
     assert redis.store["spend:key:hashed"] == pytest.approx(9.9)
     assert redis.store["spend:team:team"] == pytest.approx(2.1)
-
-
-@pytest.mark.asyncio
-async def test_update_cache_reads_an_object_redis_gained_right_after_a_batch_read_missed_it(monkeypatch):
-    """DualCache throttles repeated batch reads of a key that just missed; the per-object GET update_cache used to
-    issue never did, so its batched read must not either."""
-    from litellm.caching.dual_cache import DualCache
-
-    redis = CountingRedis()
-    cache = DualCache(redis_cache=redis)
-    monkeypatch.setattr(ps, "user_api_key_cache", cache)
-    assert await cache.async_batch_get_cache(keys=["team_id:team"]) == [None]
-    redis.store["team_id:team"] = {"spend": 1.0}
-    assert await cache.async_batch_get_cache(keys=["team_id:team"]) == [None]
-
-    assert await ps._read_update_cache_values(keys=["team_id:team"], parent_otel_span=None) == {
-        "team_id:team": {"spend": 1.0}
-    }
-    assert redis.commands.count("MGET team_id:team") == 2

tests/unit/proxy/test_proxy_server_endpoints_and_startup.py

@@ -6189,53 +6189,24 @@ async def test_tag_cache_update_called():
     """
     Test that update_cache updates tag cache when tags are provided.
     """
-    from litellm.caching.caching import DualCache
-    from litellm.proxy.proxy_server import user_api_key_cache
+    from litellm.caching.in_memory_cache import InMemoryCache
+    from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 
-    cache = DualCache()
+    cache = UserApiKeyCache(in_memory_cache=InMemoryCache())
+    cache.in_memory_cache.set_cache("tag:test-tag", {"tag_name": "test-tag", "spend": 10.0})
 
-    setattr(
-        litellm.proxy.proxy_server,
-        "user_api_key_cache",
-        cache,
+    await litellm.proxy.proxy_server.update_cache(
+        token=None,
+        user_id=None,
+        end_user_id=None,
+        team_id=None,
+        response_cost=5.0,
+        parent_otel_span=None,
+        tags=["test-tag"],
+        cache=cache,
     )
 
-    mock_tag_obj = {
-        "tag_name": "test-tag",
-        "spend": 10.0,
-    }
-
-    set_cache_called: Final = asyncio.Event()
-
-    async def _record_set_cache(*args: object, **kwargs: object) -> None:
-        set_cache_called.set()
-
-    with patch.object(cache, "async_batch_get_cache", new=AsyncMock(return_value=[mock_tag_obj])) as mock_get_cache:
-        with patch.object(
-            cache, "async_set_cache_pipeline", new=AsyncMock(side_effect=_record_set_cache)
-        ) as mock_set_cache:
-            await litellm.proxy.proxy_server.update_cache(
-                token=None,
-                user_id=None,
-                end_user_id=None,
-                team_id=None,
-                response_cost=5.0,
-                parent_otel_span=None,
-                tags=["test-tag"],
-            )
-
-            assert await asyncio.wait_for(set_cache_called.wait(), timeout=5)
-
-            mock_get_cache.assert_awaited_once_with(keys=["tag:test-tag"], parent_otel_span=None, throttle_redis=False)
-            mock_set_cache.assert_awaited_once()
-
-            call_args = mock_set_cache.call_args
-            cache_list = call_args.kwargs["cache_list"]
-
-            assert len(cache_list) == 1
-            cache_key, cache_value = cache_list[0]
-            assert cache_key == "tag:test-tag"
-            assert cache_value["spend"] == 15.0
+    assert cache.in_memory_cache.get_cache("tag:test-tag")["spend"] == 15.0
 
 
 @pytest.mark.asyncio
@@ -6243,59 +6214,26 @@ async def test_tag_cache_update_multiple_tags():
     """
     Test that multiple tags are updated in cache.
     """
-    from litellm.caching.caching import DualCache
-    from litellm.proxy.proxy_server import user_api_key_cache
+    from litellm.caching.in_memory_cache import InMemoryCache
+    from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 
-    cache = DualCache()
+    cache = UserApiKeyCache(in_memory_cache=InMemoryCache())
+    cache.in_memory_cache.set_cache("tag:tag1", {"tag_name": "tag1", "spend": 10.0})
+    cache.in_memory_cache.set_cache("tag:tag2", {"tag_name": "tag2", "spend": 20.0})
 
-    setattr(
-        litellm.proxy.proxy_server,
-        "user_api_key_cache",
-        cache,
+    await litellm.proxy.proxy_server.update_cache(
+        token=None,
+        user_id=None,
+        end_user_id=None,
+        team_id=None,
+        response_cost=5.0,
+        parent_otel_span=None,
+        tags=["tag1", "tag2"],
+        cache=cache,
     )
 
-    mock_tag1_obj = {"tag_name": "tag1", "spend": 10.0}
-    mock_tag2_obj = {"tag_name": "tag2", "spend": 20.0}
-
-    async def mock_get_cache_side_effect(keys, **kwargs):
-        return [{"tag:tag1": mock_tag1_obj, "tag:tag2": mock_tag2_obj}.get(key) for key in keys]
-
-    set_cache_called: Final = asyncio.Event()
-
-    async def _record_set_cache(*args: object, **kwargs: object) -> None:
-        set_cache_called.set()
-
-    with patch.object(
-        cache, "async_batch_get_cache", new=AsyncMock(side_effect=mock_get_cache_side_effect)
-    ) as mock_get_cache:
-        with patch.object(
-            cache, "async_set_cache_pipeline", new=AsyncMock(side_effect=_record_set_cache)
-        ) as mock_set_cache:
-            await litellm.proxy.proxy_server.update_cache(
-                token=None,
-                user_id=None,
-                end_user_id=None,
-                team_id=None,
-                response_cost=5.0,
-                parent_otel_span=None,
-                tags=["tag1", "tag2"],
-            )
-
-            assert await asyncio.wait_for(set_cache_called.wait(), timeout=5)
-
-            mock_get_cache.assert_awaited_once_with(keys=["tag:tag1", "tag:tag2"], parent_otel_span=None, throttle_redis=False)
-            mock_set_cache.assert_awaited_once()
-
-            call_args = mock_set_cache.call_args
-            cache_list = call_args.kwargs["cache_list"]
-
-            assert len(cache_list) == 2
-
-            tag_updates = {cache_key: cache_value for cache_key, cache_value in cache_list}
-            assert "tag:tag1" in tag_updates
-            assert "tag:tag2" in tag_updates
-            assert tag_updates["tag:tag1"]["spend"] == 15.0
-            assert tag_updates["tag:tag2"]["spend"] == 25.0
+    assert cache.in_memory_cache.get_cache("tag:tag1")["spend"] == 15.0
+    assert cache.in_memory_cache.get_cache("tag:tag2")["spend"] == 25.0
 
 
 @pytest.mark.asyncio
@@ -6306,41 +6244,25 @@ async def test_update_cache_pipeline_honors_user_api_key_cache_ttl():
     a hardcoded 60s, otherwise every priced request resets an active key's cache
     entry back to 60s and the configured TTL is never observed.
     """
-    from litellm.caching.caching import DualCache
+    from litellm.caching.in_memory_cache import InMemoryCache
+    from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 
-    original_cache = litellm.proxy.proxy_server.user_api_key_cache
-    cache = DualCache(default_in_memory_ttl=300)
-    setattr(litellm.proxy.proxy_server, "user_api_key_cache", cache)
-    set_cache_called: Final = asyncio.Event()
+    cache = UserApiKeyCache(in_memory_cache=InMemoryCache(clock=lambda: 1000.0), default_in_memory_ttl=300)
+    cache.in_memory_cache.cache_dict["tag:active-tag"] = {"tag_name": "active-tag", "spend": 1.0}
 
-    async def _record_set_cache(*args: object, **kwargs: object) -> None:
-        set_cache_called.set()
+    await litellm.proxy.proxy_server.update_cache(
+        token=None,
+        user_id=None,
+        end_user_id=None,
+        team_id=None,
+        response_cost=5.0,
+        parent_otel_span=None,
+        tags=["active-tag"],
+        cache=cache,
+    )
 
-    try:
-        with patch.object(
-            cache,
-            "async_batch_get_cache",
-            new=AsyncMock(return_value=[{"tag_name": "active-tag", "spend": 1.0}]),
-        ):
-            with patch.object(
-                cache, "async_set_cache_pipeline", new=AsyncMock(side_effect=_record_set_cache)
-            ) as mock_set_cache:
-                await litellm.proxy.proxy_server.update_cache(
-                    token=None,
-                    user_id=None,
-                    end_user_id=None,
-                    team_id=None,
-                    response_cost=5.0,
-                    parent_otel_span=None,
-                    tags=["active-tag"],
-                )
-
-                assert await asyncio.wait_for(set_cache_called.wait(), timeout=5)
-
-                mock_set_cache.assert_awaited_once()
-                assert mock_set_cache.call_args.kwargs["ttl"] == 300
-    finally:
-        setattr(litellm.proxy.proxy_server, "user_api_key_cache", original_cache)
+    assert cache.in_memory_cache.ttl_dict["tag:active-tag"] == 1300.0
+    assert cache.in_memory_cache.get_cache("tag:active-tag")["spend"] == 6.0
 
 
 @pytest.mark.asyncio
@@ -6393,63 +6315,6 @@ async def test_spend_tracking_never_writes_the_auth_object_back():
         setattr(litellm.proxy.proxy_server, "user_api_key_cache", original_cache)
 
 
-@pytest.mark.asyncio
-async def test_update_cache_global_proxy_spend_scalar_stays_shared():
-    """
-    The proxy-wide spend estimate must keep flowing to Redis when the spend
-    writeback goes per-pod: the global max_budget check reads the
-    ``{litellm_proxy_admin_name}:spend`` cache entry between authoritative DB
-    reloads, so keeping it pod-local would let traffic spread across replicas
-    exceed the proxy budget by roughly a factor of the replica count within a
-    cache TTL. Sharing this scalar is safe because it carries no limits or
-    permissions, so it cannot resurrect an invalidated auth blob.
-    """
-    from litellm.caching.caching import DualCache
-
-    admin_name = litellm.proxy.proxy_server.litellm_proxy_admin_name
-    global_key = "{}:spend".format(admin_name)
-
-    def fake_get(key):
-        if key == "user-lit":
-            return {"user_id": "user-lit", "spend": 1.0}
-        if key == global_key:
-            return 10.0
-        return None
-
-    async def fake_batch_get(keys, **kwargs):
-        return [fake_get(key) for key in keys]
-
-    original_cache = litellm.proxy.proxy_server.user_api_key_cache
-    cache = DualCache(default_in_memory_ttl=300)
-    setattr(litellm.proxy.proxy_server, "user_api_key_cache", cache)
-    try:
-        with patch.object(cache, "async_batch_get_cache", new=AsyncMock(side_effect=fake_batch_get)):
-            with patch.object(cache, "async_set_cache_pipeline", new=AsyncMock()) as mock_set_cache:
-                await litellm.proxy.proxy_server.update_cache(
-                    token=None,
-                    user_id="user-lit",
-                    end_user_id=None,
-                    team_id=None,
-                    response_cost=5.0,
-                    parent_otel_span=None,
-                )
-
-                pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
-                if pending:
-                    await asyncio.wait(pending, timeout=5)
-
-                calls = mock_set_cache.await_args_list
-                local_keys = [k for c in calls if c.kwargs.get("local_only") is True for k, _ in c.kwargs["cache_list"]]
-                shared_keys = [
-                    k for c in calls if c.kwargs.get("local_only") is not True for k, _ in c.kwargs["cache_list"]
-                ]
-                assert "user-lit" in local_keys
-                assert global_key not in local_keys
-                assert shared_keys == [global_key]
-    finally:
-        setattr(litellm.proxy.proxy_server, "user_api_key_cache", original_cache)
-
-
 @pytest.mark.asyncio
 async def test_init_sso_settings_in_db():
     """
@@ -15720,46 +15585,3 @@ async def test_spend_capture_rate_check_job_clears_the_gauge_once_the_setting_is
         call(api_provider="openai", capture_rate=None),
     ]
 
-
-@pytest.mark.asyncio
-async def test_update_cache_reads_and_writes_declare_the_auth_objects_key_family():
-    """The post-call spend write-back reads and rewrites the cached auth objects, so its
-    Redis spans must read ``redis.mget auth_objects`` / ``redis.set auth_objects`` (the key
-    family the auth phase declares) and the global spend scalar ``redis.set spend_counters``,
-    never a bare ``redis.mget`` with no owner."""
-    from litellm.caching.caching import DualCache
-
-    original_cache = litellm.proxy.proxy_server.user_api_key_cache
-    cache = DualCache()
-    setattr(litellm.proxy.proxy_server, "user_api_key_cache", cache)
-    seen: list[tuple[str, str | None]] = []
-
-    async def _mget(keys, **_kwargs):
-        seen.append(("mget", current_service_target()))
-        return [{"user_id": "u1", "spend": 1.0} for _ in keys]
-
-    async def _set_pipeline(**_kwargs):
-        seen.append(("set", current_service_target()))
-
-    try:
-        with (
-            patch.object(cache, "async_batch_get_cache", new=AsyncMock(side_effect=_mget)),
-            patch.object(cache, "async_set_cache_pipeline", new=AsyncMock(side_effect=_set_pipeline)),
-        ):
-            await litellm.proxy.proxy_server.update_cache(
-                token=None,
-                user_id="u1",
-                end_user_id=None,
-                team_id=None,
-                response_cost=2.0,
-                parent_otel_span=None,
-            )
-            pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
-            if pending:
-                await asyncio.wait(pending, timeout=5)
-    finally:
-        setattr(litellm.proxy.proxy_server, "user_api_key_cache", original_cache)
-
-    assert seen, "update_cache must touch the cache for a priced user request"
-    assert {target for _, target in seen} == {"auth_objects"}
-    assert current_service_target() is None

tests/unit/test_internal_context.py

@@ -87,10 +87,10 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset(
 )
 
 _CACHE_CALL: Final = re.compile(
-    r"\.(?:async_)?(?:get_cache|set_cache|batch_get_cache|batch_get_cache_shared|increment_cache|increment"
+    r"\.(?:async_)?(?:get_cache|set_cache|batch_get_cache|batch_get_cache_shared|increment_cache|increment_if_present|increment"
     r"|set_cache_pipeline|set_cache_pipeline_with_ttls|set_cache_sadd|delete_cache|batch_set_cache|increment_pipeline"
     r"|rpush|lpop|scan_iter|get_ttl|mget)\("
-    r"|\b(?:reserve_redis_batch_reads|declare_batch_get|_prepare_batch_get)\("
+    r"|\b(?:reserve_redis_batch_reads|_prepare_batch_get)\("
     r"|\bbatch\.(?:set|delete|script|increment)\("
 )
 _DECLARES_TARGET: Final = re.compile(r"\b(?:with_service_target|service_target|response_cache_phase)\(")

tests/unit/test_main.py

@@ -23,6 +23,7 @@ import respx
 from openai import APITimeoutError, AsyncOpenAI
 from openai._legacy_response import LegacyAPIResponse
 from openai.types.chat.chat_completion import ChatCompletion
+from pydantic import TypeAdapter
 
 import litellm
 from litellm import (
@@ -3937,6 +3938,100 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai():
         assert request_body["prompt_cache_key"] == "test-cache-key"
 
 
+_TOOL: Final[Mapping[str, object]] = {"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}
+
+# Client settings ride in OPENAI_CHAT_COMPLETION_PARAMS but configure the HTTP client, never the body.
+_OPENAI_CLIENT_SETTINGS: Final = frozenset(
+    {
+        "api_base",
+        "api_key",
+        "api_version",
+        "base_url",
+        "default_headers",
+        "deployment_id",
+        "extra_headers",
+        "max_retries",
+        "organization",
+        "request_timeout",
+        "timeout",
+    }
+)
+
+# Accepted by completion() for other providers, never sent to OpenAI's /chat/completions body.
+_NOT_OPENAI_BODY_PARAMS: Final = frozenset({"include_server_side_tool_invocations", "thinking"})
+
+# param -> (model that accepts it, the call kwargs that carry it)
+_OPENAI_BODY_PARAM_CASES: Final[Mapping[str, tuple[str, Mapping[str, object]]]] = {
+    "audio": ("openai/gpt-4o", {"audio": {"voice": "alloy", "format": "wav"}}),
+    "frequency_penalty": ("openai/gpt-4o", {"frequency_penalty": 0.5}),
+    "function_call": ("openai/gpt-4o", {"function_call": "auto"}),
+    "functions": ("openai/gpt-4o", {"functions": [_TOOL["function"]]}),
+    "logit_bias": ("openai/gpt-4o", {"logit_bias": {"1": 1}}),
+    "logprobs": ("openai/gpt-4o", {"logprobs": True}),
+    "max_completion_tokens": ("openai/gpt-4o", {"max_completion_tokens": 7}),
+    "max_tokens": ("openai/gpt-4o", {"max_tokens": 7}),
+    "modalities": ("openai/gpt-4o", {"modalities": ["text"]}),
+    "n": ("openai/gpt-4o", {"n": 2}),
+    "parallel_tool_calls": ("openai/gpt-4o", {"tools": [_TOOL], "parallel_tool_calls": False}),
+    "prediction": ("openai/gpt-4o", {"prediction": {"type": "content", "content": "x"}}),
+    "presence_penalty": ("openai/gpt-4o", {"presence_penalty": 0.5}),
+    "prompt_cache_key": ("openai/gpt-4o", {"prompt_cache_key": "k"}),
+    "prompt_cache_retention": ("openai/gpt-5.5", {"prompt_cache_retention": "24h"}),
+    "reasoning_effort": ("openai/gpt-5.5", {"reasoning_effort": "low"}),
+    "response_format": ("openai/gpt-4o", {"response_format": {"type": "json_object"}}),
+    "safety_identifier": ("openai/gpt-4o", {"safety_identifier": "s"}),
+    "seed": ("openai/gpt-4o", {"seed": 3}),
+    "service_tier": ("openai/gpt-4o", {"service_tier": "priority"}),
+    "stop": ("openai/gpt-4o", {"stop": ["x"]}),
+    "store": ("openai/gpt-4o", {"store": False}),
+    "stream": ("openai/gpt-4o", {"stream": True}),
+    "stream_options": ("openai/gpt-4o", {"stream": True, "stream_options": {"include_usage": True}}),
+    "temperature": ("openai/gpt-4o", {"temperature": 0.5}),
+    "tool_choice": ("openai/gpt-4o", {"tools": [_TOOL], "tool_choice": "auto"}),
+    "tools": ("openai/gpt-4o", {"tools": [_TOOL]}),
+    "top_logprobs": ("openai/gpt-4o", {"logprobs": True, "top_logprobs": 2}),
+    "top_p": ("openai/gpt-4o", {"top_p": 0.5}),
+    "user": ("openai/gpt-4o", {"user": "u1"}),
+    "verbosity": ("openai/gpt-5.5", {"verbosity": "low"}),
+    "web_search_options": ("openai/gpt-4o", {"web_search_options": {"search_context_size": "low"}}),
+}
+
+
+def test_every_openai_body_param_has_a_forwarding_case():
+    body_params: Final = frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS
+    assert frozenset(_OPENAI_BODY_PARAM_CASES) == body_params, (
+        "a param added to OPENAI_CHAT_COMPLETION_PARAMS needs a case in _OPENAI_BODY_PARAM_CASES proving "
+        "completion() sends it to OpenAI; prompt_cache_retention (LIT-9758) and store/prompt_cache_key "
+        "(#33184) were listed as supported but silently dropped"
+    )
+
+
+_REQUEST_BODY: Final = TypeAdapter(dict[str, object])
+
+
+@pytest.mark.parametrize("param", sorted(_OPENAI_BODY_PARAM_CASES))
+def test_completion_sends_openai_body_param_to_openai(
+    param: str, respx_mock: respx.MockRouter, openai_api_response: Mapping[str, object]
+):
+    api_base: Final = "http://localhost:12346/v1"
+    route: Final = respx_mock.post(f"{api_base}/chat/completions").mock(
+        return_value=httpx.Response(status_code=200, json=openai_api_response)
+    )
+    model, call_kwargs = _OPENAI_BODY_PARAM_CASES[param]
+
+    litellm.completion(
+        model=model,
+        messages=[{"role": "user", "content": "hi"}],
+        api_base=api_base,
+        api_key="fake_openai_api_key",
+        **call_kwargs,
+    )
+
+    assert route.call_count == 1
+    sent: Final = _REQUEST_BODY.validate_json(route.calls.last.request.content)
+    assert sent.get(param) == call_kwargs[param], f"{param} did not reach the OpenAI request body: {sent}"
+
+
 def test_completion_omits_store_and_prompt_cache_key_when_not_passed():
     """
     When store and prompt_cache_key are not passed, they must not appear in the