From f3513bca9e5bee227e5603c1d693c31d40d4e86a Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 17:04:41 +0000
Subject: [PATCH 001/203] fix(bedrock): resolve AWS region and secret lookups
off the event loop (#47331)
* fix(bedrock): resolve AWS region and secret lookups off the event loop
Bedrock request paths that resolve their region, endpoint or bearer token
inside the async handler now do it in the AWS signing executor, so a slow
secret manager read no longer stalls every other request on the worker
* fix(bedrock): type the STS client kwargs and claude platform environment override
Keeps the basedpyright counts at or below the merge base after the off-loop wraps, by typing the STS client kwargs as a TypedDict, the claude platform environment override with concrete dicts, and the realtime region call through the typed getter
* test(bedrock): detect event loop stalls without wall-clock timing
The Vault fake now answers a read only after the test loop runs a callback, so a read made on the loop thread is caught deterministically instead of by a sleep and a heartbeat threshold
---
litellm/llms/bedrock/base_aws_llm.py | 37 +++--
.../messages_transformation.py | 34 +++-
litellm/llms/bedrock/count_tokens/handler.py | 6 +-
.../bedrock/count_tokens/mantle_handler.py | 6 +-
litellm/llms/bedrock/realtime/handler.py | 5 +-
litellm/llms/custom_httpx/llm_http_handler.py | 74 ++++++---
.../guardrail_hooks/bedrock_guardrails.py | 12 +-
.../llm_passthrough_endpoints.py | 9 +-
.../test_bedrock_count_tokens_handler.py | 29 ++++
...est_bedrock_mantle_count_tokens_handler.py | 20 +++
tests/unit/llms/bedrock/event_loop_probe.py | 38 ++++-
.../realtime/test_bedrock_realtime_handler.py | 22 +++
tests/unit/llms/bedrock/test_base_aws_llm.py | 6 +-
.../bedrock/test_claude_platform_provider.py | 27 ++++
.../test_bedrock_mantle_transformation.py | 6 +-
.../custom_httpx/test_llm_http_handler.py | 153 +++++++++++++++++-
.../test_bedrock_guardrails.py | 42 +++++
.../test_llm_pass_through_endpoints.py | 43 +++++
18 files changed, 496 insertions(+), 73 deletions(-)
diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py
index f1426aa9375..9591b962ab5 100644
--- a/litellm/llms/bedrock/base_aws_llm.py
+++ b/litellm/llms/bedrock/base_aws_llm.py
@@ -80,6 +80,12 @@ class _AssumeRoleParams(TypedDict):
Tags: ReadOnly[NotRequired[tuple[AwsSessionTag, ...]]]
+class _StsClientKwargs(TypedDict):
+ verify: ReadOnly[bool | str]
+ endpoint_url: ReadOnly[NotRequired[str]]
+ region_name: ReadOnly[NotRequired[str]]
+
+
def _assume_role_params(
aws_role_name: str,
aws_session_name: str,
@@ -275,7 +281,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
"aws_session_tags",
]
- def _get_ssl_verify(self, ssl_verify: bool | str | None = None):
+ def _get_ssl_verify(self, ssl_verify: bool | str | None = None) -> bool | str:
"""
Get SSL verification setting for boto3 clients.
@@ -875,15 +881,18 @@ class BaseAWSLLM(SignsRequestsWithAWS):
aws_sts_endpoint: str | None = None,
ssl_verify: bool | str | None = None,
aws_region_name: str | None = None,
- ) -> dict:
+ ) -> _StsClientKwargs:
"""STS client kwargs with aligned endpoint_url and region_name (SigV4)."""
- kwargs: Final[dict] = {"verify": self._get_ssl_verify(ssl_verify)}
- if aws_sts_endpoint is not None:
- kwargs["endpoint_url"] = aws_sts_endpoint
- sts_region: Final = self._resolve_sts_region(aws_sts_endpoint, aws_region_name)
- if sts_region is not None:
- kwargs["region_name"] = sts_region
- return kwargs
+ verify: Final = self._get_ssl_verify(ssl_verify)
+ match (aws_sts_endpoint, self._resolve_sts_region(aws_sts_endpoint, aws_region_name)):
+ case (None, None):
+ return _StsClientKwargs(verify=verify)
+ case (None, region):
+ return _StsClientKwargs(verify=verify, region_name=region)
+ case (endpoint, None):
+ return _StsClientKwargs(verify=verify, endpoint_url=endpoint)
+ case (endpoint, region):
+ return _StsClientKwargs(verify=verify, endpoint_url=endpoint, region_name=region)
def get_aws_region_name_for_non_llm_api_calls(
self,
@@ -1813,13 +1822,13 @@ async def run_aws_signing(
)
-async def sign_request_off_loop_if_aws(
+async def run_off_loop_if_aws(
provider_config: object,
- sign_request: Callable[_SignParams, _SignedRequest],
+ call: Callable[_SignParams, _SignedRequest],
/,
*args: _SignParams.args,
- **kwargs: _SignParams.kwargs, # kwargs-ok: ParamSpec forwarding keeps the wrapped sign_request signature
+ **kwargs: _SignParams.kwargs, # kwargs-ok: ParamSpec forwarding keeps the wrapped call's signature
) -> _SignedRequest:
if isinstance(provider_config, SignsRequestsWithAWS):
- return await run_aws_signing(sign_request, *args, **kwargs)
- return sign_request(*args, **kwargs)
+ return await run_aws_signing(call, *args, **kwargs)
+ return call(*args, **kwargs)
diff --git a/litellm/llms/bedrock/claude_platform/messages_transformation.py b/litellm/llms/bedrock/claude_platform/messages_transformation.py
index 1469f6a6935..57ebaa0ca71 100644
--- a/litellm/llms/bedrock/claude_platform/messages_transformation.py
+++ b/litellm/llms/bedrock/claude_platform/messages_transformation.py
@@ -1,10 +1,11 @@
-from typing import Any, Final
+from typing import Final
import litellm
from litellm.llms.anthropic.pass_through.messages.transformation import (
DEFAULT_ANTHROPIC_API_VERSION,
AnthropicMessagesConfig,
)
+from litellm.llms.bedrock.base_aws_llm import run_aws_signing
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
@@ -22,14 +23,14 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM
def validate_anthropic_messages_environment(
self,
- headers: dict,
+ headers: dict[str, str], # mutable-ok: overrides the base signature
model: str,
- messages: list[Any],
- optional_params: dict,
- litellm_params: dict,
+ messages: list[object], # mutable-ok: overrides the base signature
+ optional_params: dict[str, object], # mutable-ok: overrides the base signature
+ litellm_params: dict[str, object], # mutable-ok: overrides the base signature
api_key: str | None = None,
api_base: str | None = None,
- ) -> tuple[dict, str | None]:
+ ) -> tuple[dict[str, str], str | None]: # mutable-ok: overrides the base signature
workspace_id: Final = self._get_workspace_id(optional_params, litellm_params)
if workspace_id is None:
raise litellm.AuthenticationError(
@@ -66,6 +67,27 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM
api_base,
)
+ async def avalidate_anthropic_messages_environment(
+ self,
+ headers: dict[str, str], # mutable-ok: mirrors validate_anthropic_messages_environment
+ model: str,
+ messages: list[object], # mutable-ok: mirrors validate_anthropic_messages_environment
+ optional_params: dict[str, object], # mutable-ok: mirrors validate_anthropic_messages_environment
+ litellm_params: dict[str, object], # mutable-ok: mirrors validate_anthropic_messages_environment
+ api_key: str | None = None,
+ api_base: str | None = None,
+ ) -> tuple[dict[str, str], str | None]: # mutable-ok: mirrors validate_anthropic_messages_environment
+ return await run_aws_signing(
+ self.validate_anthropic_messages_environment,
+ headers=headers,
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ api_key=api_key,
+ api_base=api_base,
+ )
+
def transform_anthropic_messages_request(
self,
model: str,
diff --git a/litellm/llms/bedrock/count_tokens/handler.py b/litellm/llms/bedrock/count_tokens/handler.py
index 3bae1f22490..5f8f478e4cd 100644
--- a/litellm/llms/bedrock/count_tokens/handler.py
+++ b/litellm/llms/bedrock/count_tokens/handler.py
@@ -48,7 +48,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model)
# Get AWS region using existing LiteLLM function
- aws_region_name: Final = self.get_aws_region_name(
+ aws_region_name: Final = await run_aws_signing(
+ self.get_aws_region_name,
optional_params=litellm_params,
model=resolved_model,
model_id=None,
@@ -64,7 +65,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
# Get endpoint URL using simplified function
api_base: Final = litellm_params.get("api_base", None)
aws_bedrock_runtime_endpoint: Final = litellm_params.get("aws_bedrock_runtime_endpoint", None)
- endpoint_url: Final = self.get_bedrock_count_tokens_endpoint(
+ endpoint_url: Final = await run_aws_signing(
+ self.get_bedrock_count_tokens_endpoint,
model=resolved_model,
aws_region_name=aws_region_name,
api_base=api_base,
diff --git a/litellm/llms/bedrock/count_tokens/mantle_handler.py b/litellm/llms/bedrock/count_tokens/mantle_handler.py
index 4b1a6a75a7a..14edd39c386 100644
--- a/litellm/llms/bedrock/count_tokens/mantle_handler.py
+++ b/litellm/llms/bedrock/count_tokens/mantle_handler.py
@@ -54,10 +54,10 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
) -> dict[str, JsonValue]:
try:
request: Final = _COUNT_REQUEST.validate_python(request_data)
- aws_region_name: Final = self.get_aws_region_name(
- optional_params=litellm_params, model=resolved_model, model_id=None
+ aws_region_name: Final = await run_aws_signing(
+ self.get_aws_region_name, optional_params=litellm_params, model=resolved_model, model_id=None
)
- endpoint_url: Final = self.get_mantle_count_tokens_endpoint(aws_region_name)
+ endpoint_url: Final = await run_aws_signing(self.get_mantle_count_tokens_endpoint, aws_region_name)
body: Final = self._anthropic_config.transform_request_to_count_tokens(
model=resolved_model,
messages=request["messages"],
diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py
index 3e7c2f7eee9..e9b2f26e977 100644
--- a/litellm/llms/bedrock/realtime/handler.py
+++ b/litellm/llms/bedrock/realtime/handler.py
@@ -337,10 +337,7 @@ class BedrockRealtime(BaseAWSLLM):
# Get AWS region
if aws_region_name is None:
- optional_params: Final = {
- "aws_region_name": aws_region_name,
- }
- aws_region_name = self._get_aws_region_name(optional_params, model)
+ aws_region_name = await run_aws_signing(self.get_aws_region_name, {}, model)
# Get endpoint URL
if api_base is not None:
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 48858dfa441..b1e55375f16 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -104,7 +104,7 @@ from litellm.llms.base_llm.vector_store_files.transformation import (
BaseVectorStoreFilesConfig,
)
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
-from litellm.llms.bedrock.base_aws_llm import SignsRequestsWithAWS, run_aws_signing, sign_request_off_loop_if_aws
+from litellm.llms.bedrock.base_aws_llm import SignsRequestsWithAWS, run_aws_signing, run_off_loop_if_aws
from litellm.llms.custom_httpx.container_handler import raise_for_error_status
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@@ -2085,14 +2085,18 @@ class BaseLLMHTTPHandler:
raise ValueError(f"No provider config found for provider: {custom_llm_provider}")
# Validate environment and get headers
- headers = provider_config.validate_environment(
+ headers = await run_off_loop_if_aws(
+ provider_config,
+ provider_config.validate_environment,
api_key=api_key,
api_base=api_base,
headers=headers or {},
)
# Transform the request first
- data: Final = provider_config.transform_search_request(
+ data: Final = await run_off_loop_if_aws(
+ provider_config,
+ provider_config.transform_search_request,
query=query,
optional_params=optional_params,
api_key=api_key,
@@ -2108,7 +2112,7 @@ class BaseLLMHTTPHandler:
api_key=api_key,
)
- signed_headers, signed_json_body = await sign_request_off_loop_if_aws(
+ signed_headers, signed_json_body = await run_off_loop_if_aws(
provider_config,
provider_config.sign_request,
headers=headers,
@@ -2236,7 +2240,7 @@ class BaseLLMHTTPHandler:
)
provider_config.transform_anthropic_messages_request_on_http_error(e=e, request_data=request_body)
recovered_errors = (*recovered_errors, e)
- headers, signed_json_body = await sign_request_off_loop_if_aws(
+ headers, signed_json_body = await run_off_loop_if_aws(
provider_config,
provider_config.sign_request,
headers=headers,
@@ -2405,7 +2409,9 @@ class BaseLLMHTTPHandler:
logging_obj.model_call_details.update(request_body)
# Make the request
- request_url: Final = anthropic_messages_provider_config.get_complete_url(
+ request_url: Final = await run_off_loop_if_aws(
+ anthropic_messages_provider_config,
+ anthropic_messages_provider_config.get_complete_url,
api_base=api_base,
api_key=api_key,
model=model,
@@ -2416,7 +2422,7 @@ class BaseLLMHTTPHandler:
stream=stream,
)
- headers, signed_json_body = await sign_request_off_loop_if_aws(
+ headers, signed_json_body = await run_off_loop_if_aws(
anthropic_messages_provider_config,
anthropic_messages_provider_config.sign_request,
headers=headers,
@@ -2967,7 +2973,9 @@ class BaseLLMHTTPHandler:
stream = response_api_optional_request_params.get("stream", False)
caller_requested_stream: Final = bool(stream or (extra_body or {}).get("stream"))
- api_base: Final = responses_api_provider_config.get_complete_url(
+ api_base: Final = await run_off_loop_if_aws(
+ responses_api_provider_config,
+ responses_api_provider_config.get_complete_url,
api_base=litellm_params.api_base,
litellm_params=dict(litellm_params),
)
@@ -3009,7 +3017,7 @@ class BaseLLMHTTPHandler:
fake_stream=fake_stream,
)
- headers, signed_body = await sign_request_off_loop_if_aws(
+ headers, signed_body = await run_off_loop_if_aws(
responses_api_provider_config,
responses_api_provider_config.sign_request,
headers=validated_headers,
@@ -3856,7 +3864,9 @@ class BaseLLMHTTPHandler:
litellm_params=litellm_params,
api_key=api_key,
)
- complete_api_base: Final = provider_config.get_complete_file_url(
+ complete_api_base: Final = await run_off_loop_if_aws(
+ provider_config,
+ provider_config.get_complete_file_url,
api_base=api_base,
api_key=api_key,
model="",
@@ -3867,7 +3877,9 @@ class BaseLLMHTTPHandler:
if not complete_api_base:
raise ValueError("api_base is required for create_file")
return await self.async_create_file(
- transformed_request=provider_config.transform_create_file_request(
+ transformed_request=await run_off_loop_if_aws(
+ provider_config,
+ provider_config.transform_create_file_request,
model="",
create_file_data=create_file_data,
litellm_params=litellm_params,
@@ -4334,7 +4346,9 @@ class BaseLLMHTTPHandler:
litellm_params=litellm_params,
api_key=api_key,
)
- complete_api_base: Final = provider_config.get_complete_batch_url(
+ complete_api_base: Final = await run_off_loop_if_aws(
+ provider_config,
+ provider_config.get_complete_batch_url,
api_base=api_base,
api_key=api_key,
model=model,
@@ -4345,7 +4359,9 @@ class BaseLLMHTTPHandler:
if not complete_api_base:
raise ValueError("api_base is required for create_batch")
return await self.async_create_batch(
- transformed_request=provider_config.transform_create_batch_request(
+ transformed_request=await run_off_loop_if_aws(
+ provider_config,
+ provider_config.transform_create_batch_request,
model=model,
create_batch_data=create_batch_data,
litellm_params=litellm_params,
@@ -4818,7 +4834,9 @@ class BaseLLMHTTPHandler:
**(extra_headers or {}),
}
- api_base: Final = responses_api_provider_config.get_complete_url(
+ api_base: Final = await run_off_loop_if_aws(
+ responses_api_provider_config,
+ responses_api_provider_config.get_complete_url,
api_base=litellm_params.api_base,
litellm_params=dict(litellm_params),
)
@@ -4836,7 +4854,7 @@ class BaseLLMHTTPHandler:
)
data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
- headers, signed_body = await sign_request_off_loop_if_aws(
+ headers, signed_body = await run_off_loop_if_aws(
responses_api_provider_config,
responses_api_provider_config.sign_request,
headers=validated_headers,
@@ -4962,7 +4980,9 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Get URL and params from provider config
- url, params = provider_config.transform_retrieve_file_request(
+ url, params = await run_off_loop_if_aws(
+ provider_config,
+ provider_config.transform_retrieve_file_request,
file_id=file_id,
optional_params={},
litellm_params=litellm_params,
@@ -5089,7 +5109,9 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Get URL and params from provider config
- url, params = provider_config.transform_delete_file_request(
+ url, params = await run_off_loop_if_aws(
+ provider_config,
+ provider_config.transform_delete_file_request,
file_id=file_id,
optional_params={},
litellm_params=litellm_params,
@@ -5214,7 +5236,9 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Get URL and params from provider config
- url, params = provider_config.transform_list_files_request(
+ url, params = await run_off_loop_if_aws(
+ provider_config,
+ provider_config.transform_list_files_request,
purpose=purpose,
optional_params={},
litellm_params=litellm_params,
@@ -5298,7 +5322,9 @@ class BaseLLMHTTPHandler:
)
yield page_files[: MAX_FILE_LIST_LIMIT - listed_count]
listed_count += len(page_files)
- next_request = self._next_listing_request(latest_page, provider_config, litellm_params, listed_count)
+ next_request = await run_off_loop_if_aws(
+ provider_config, self._next_listing_request, latest_page, provider_config, litellm_params, listed_count
+ )
if next_request is None:
return
url, params = next_request
@@ -5524,7 +5550,9 @@ class BaseLLMHTTPHandler:
headers: dict,
logging_obj: LiteLLMLoggingObj,
) -> "_PreparedFileContentRequest":
- url, params = provider_config.transform_file_content_request(
+ url, params = await run_off_loop_if_aws(
+ provider_config,
+ provider_config.transform_file_content_request,
file_content_request=file_content_request,
optional_params={},
litellm_params=litellm_params,
@@ -10332,7 +10360,9 @@ class BaseLLMHTTPHandler:
if extra_headers:
headers.update(extra_headers)
- api_base: Final = vector_store_provider_config.get_complete_url(
+ api_base: Final = await run_off_loop_if_aws(
+ vector_store_provider_config,
+ vector_store_provider_config.get_complete_url,
api_base=litellm_params.api_base,
litellm_params=dict(litellm_params),
)
@@ -10366,7 +10396,7 @@ class BaseLLMHTTPHandler:
)
all_optional_params: Final[dict[str, object]] = dict(litellm_params)
all_optional_params.update(vector_store_search_optional_params or {})
- headers, signed_json_body = await sign_request_off_loop_if_aws(
+ headers, signed_json_body = await run_off_loop_if_aws(
vector_store_provider_config,
vector_store_provider_config.sign_request,
headers=headers,
diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
index 12b8e728b4e..adda465ad95 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
@@ -796,7 +796,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
# logic becomes shared across providers.
#### CALL HOOKS - proxy only ####
- def _load_credentials(self, bearer_token: str | None = None):
+ def _load_credentials(self, api_key: str | None = None):
try:
from botocore.credentials import Credentials
except ImportError:
@@ -820,7 +820,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
credentials: Final[Credentials | None] = (
None
- if bearer_token is not None
+ if bedrock_bearer_token(api_key) is not None
else self.get_credentials(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
@@ -1017,9 +1017,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
start_time: "datetime",
allow_chunking: bool,
) -> BedrockGuardrailResponse:
- credentials, aws_region_name = await run_aws_signing(
- self._load_credentials, bearer_token=bedrock_bearer_token(api_key)
- )
+ credentials, aws_region_name = await run_aws_signing(self._load_credentials, api_key=api_key)
completed_chunk_usages: Final[list[BedrockGuardrailUsage]] = [] # mutable-ok: billed-chunk usage accumulator
try:
@@ -1960,9 +1958,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
return BedrockGuardrailResponse()
api_key: Final[str | None] = request_data.get("api_key") if request_data else None
- credentials, aws_region_name = await run_aws_signing(
- self._load_credentials, bearer_token=bedrock_bearer_token(api_key)
- )
+ credentials, aws_region_name = await run_aws_signing(self._load_credentials, api_key=api_key)
body: Final[dict[str, object]] = {"messages": checks_messages, "checks": self.checks}
prepared_request: Final = await run_aws_signing(
diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
index d9d3b6f8c92..16fbd1de376 100644
--- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
@@ -1459,7 +1459,6 @@ async def bedrock_proxy_route(
"""
create_request_copy(request)
- aws_region_name: Final = get_secret_str(secret_name="AWS_REGION_NAME")
if not _is_bedrock_agent_runtime_route(endpoint=endpoint):
return await bedrock_llm_proxy_route(
endpoint=endpoint,
@@ -1474,6 +1473,10 @@ async def bedrock_proxy_route(
detail="bedrock-agent-runtime pass-through is disabled on this proxy.",
)
+ from litellm.llms.bedrock.base_aws_llm import run_aws_signing, sign_aws_json_post
+ from litellm.llms.bedrock.chat import BedrockConverseLLM
+
+ aws_region_name: Final = await run_aws_signing(get_secret_str, secret_name="AWS_REGION_NAME")
base_target_url: Final = f"https://bedrock-agent-runtime.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}"
encoded_endpoint = httpx.URL(endpoint).path
@@ -1488,10 +1491,6 @@ async def bedrock_proxy_route(
path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint)
)
- # Add or update query parameters
- from litellm.llms.bedrock.base_aws_llm import run_aws_signing, sign_aws_json_post
- from litellm.llms.bedrock.chat import BedrockConverseLLM
-
bedrock_llm: Final = BedrockConverseLLM()
# Assuming the body contains JSON data, parse it
try:
diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py
index cb97a62f635..5dadeff6d7c 100644
--- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py
+++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py
@@ -10,6 +10,7 @@ from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
class _ProbedCountTokensHandler(BedrockCountTokensHandler):
@@ -56,6 +57,34 @@ async def test_handle_count_tokens_request_signs_off_the_event_loop(monkeypatch)
assert probe.served_during_refresh is True
+def _count_tokens_in_us_west_2(request: httpx.Request) -> httpx.Response:
+ if request.url.host != "bedrock-runtime.us-west-2.amazonaws.com":
+ return httpx.Response(404, request=request)
+ return httpx.Response(200, json={"inputTokens": 7}, request=request)
+
+
+@pytest.mark.asyncio
+async def test_handle_count_tokens_request_looks_up_its_region_off_the_event_loop(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATESTACCESSKEY")
+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
+ monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+ vault: Final = host_in_loop_probing_vault(monkeypatch, "AWS_REGION_NAME", "AWS_BEDROCK_RUNTIME_ENDPOINT")
+
+ result = await BedrockCountTokensHandler().handle_count_tokens_request(
+ request_data={"model": "us.anthropic.claude-sonnet-5-5", "messages": [{"role": "user", "content": "hi"}]},
+ litellm_params={},
+ resolved_model="us.anthropic.claude-sonnet-5-5",
+ client=AsyncHTTPHandler(transport=httpx.MockTransport(_count_tokens_in_us_west_2)),
+ )
+
+ assert vault.read.is_set()
+ assert not vault.stalled_the_loop.is_set()
+ assert result == {"input_tokens": 7}
+
+
class TestBedrockCountTokensEndpoint:
def _make_handler(self) -> BedrockCountTokensConfig:
return BedrockCountTokensConfig()
diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_mantle_count_tokens_handler.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_mantle_count_tokens_handler.py
index 71b451f834c..2d21a3ddef5 100644
--- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_mantle_count_tokens_handler.py
+++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_mantle_count_tokens_handler.py
@@ -8,6 +8,7 @@ import pytest
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
MANTLE_COUNT_URL: Final = "https://bedrock-mantle.us-east-1.api.aws/anthropic/v1/messages/count_tokens"
LITELLM_PARAMS: Final = {
@@ -107,3 +108,22 @@ async def test_non_200_answers_raise_bedrock_error_with_mantle_status() -> None:
assert raised.value.status_code == 404
assert "does not exist" in raised.value.message
+
+
+@pytest.mark.asyncio
+async def test_region_and_endpoint_lookups_leave_the_event_loop_free(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
+ vault: Final = host_in_loop_probing_vault(monkeypatch, "AWS_REGION_NAME", "BEDROCK_MANTLE_API_BASE")
+ mantle: Final = _MantleEndpoint(200, {"input_tokens": 2177})
+
+ result = await BedrockMantleCountTokensHandler().handle_count_tokens_request(
+ request_data={**REQUEST, "model": "global.anthropic.claude-opus-5-5"},
+ litellm_params={key: value for key, value in LITELLM_PARAMS.items() if key != "aws_region_name"},
+ resolved_model="anthropic.claude-opus-5-5",
+ client=_client(mantle),
+ )
+
+ assert vault.read.is_set()
+ assert not vault.stalled_the_loop.is_set()
+ assert result == {"input_tokens": 2177}
+ assert str(mantle.only_request().url) == MANTLE_COUNT_URL
diff --git a/tests/unit/llms/bedrock/event_loop_probe.py b/tests/unit/llms/bedrock/event_loop_probe.py
index c347247ec32..cf6cd5975ed 100644
--- a/tests/unit/llms/bedrock/event_loop_probe.py
+++ b/tests/unit/llms/bedrock/event_loop_probe.py
@@ -1,4 +1,4 @@
-"""Refreshable credentials whose refresh only completes while the event loop keeps serving."""
+"""Credential refreshes and Vault reads that only complete while the event loop keeps serving."""
from __future__ import annotations
@@ -8,10 +8,15 @@ import time
from datetime import datetime, timedelta, timezone
from typing import Final
+import pytest
from botocore.credentials import RefreshableCredentials
+import litellm
+from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
+
REFRESH_RELEASE_TIMEOUT_SECONDS: Final = 2.0
REFRESH_START_TIMEOUT_SECONDS: Final = 10.0
+LOOP_SERVE_TIMEOUT_SECONDS: Final = 2.0
class EventLoopProbe:
@@ -55,3 +60,34 @@ class EventLoopProbe:
raise TimeoutError("signing finished without ever starting a credential refresh")
await asyncio.sleep(0.005)
self.loop_served.set()
+
+
+class LoopProbingVault:
+ """A Vault holding none of its hosted secrets, whose read waits for the event loop to run a callback.
+
+ A read made on the loop thread can never see that callback, so `stalled_the_loop` is set only then.
+ """
+
+ def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+ self._loop: Final = loop
+ self.read: Final = threading.Event()
+ self.stalled_the_loop: Final = threading.Event()
+
+ def sync_read_secret(self, secret_name: str) -> None:
+ self.read.set()
+ served: Final = threading.Event()
+ self._loop.call_soon_threadsafe(served.set)
+ if not served.wait(timeout=LOOP_SERVE_TIMEOUT_SECONDS):
+ self.stalled_the_loop.set()
+
+
+def host_in_loop_probing_vault(monkeypatch: pytest.MonkeyPatch, *secret_names: str) -> LoopProbingVault:
+ vault: Final = LoopProbingVault(asyncio.get_running_loop())
+ monkeypatch.setattr(litellm, "secret_manager_client", vault)
+ monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.HASHICORP_VAULT)
+ monkeypatch.setattr(
+ litellm,
+ "_key_management_settings",
+ KeyManagementSettings(access_mode="read_only", hosted_keys=list(secret_names)),
+ )
+ return vault
diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py
index a181c5a5425..ccea4ba44d7 100644
--- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py
+++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py
@@ -18,6 +18,7 @@ from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.realtime.handler import BedrockRealtime
from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
from litellm.realtime_api import main as realtime_main
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
@pytest.fixture(autouse=True)
@@ -625,6 +626,27 @@ class TestBedrockRealtimeSessionLifecycle:
assert first_event["session"]["id"] == "trace-nova-sonic"
assert first_event["session"]["model"] == "amazon.nova-sonic-v1:0"
+ @pytest.mark.asyncio
+ async def test_region_lookup_on_connect_leaves_the_event_loop_free(
+ self, stub_aws_sdk_client, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+ vault: Final = host_in_loop_probing_vault(monkeypatch, "AWS_REGION_NAME")
+ websocket: Final = RealtimeClientWS()
+
+ _ = await BedrockRealtime().async_realtime(
+ model="amazon.nova-2-sonic-v1:0",
+ websocket=websocket,
+ logging_obj=FakeLogging(),
+ aws_access_key_id="k",
+ aws_secret_access_key="s",
+ )
+
+ assert vault.read.is_set()
+ assert not vault.stalled_the_loop.is_set()
+ assert stub_aws_sdk_client["config_kwargs"]["region"] == "us-west-2"
+ assert stub_aws_sdk_client["config_kwargs"]["endpoint_uri"] == "https://bedrock-runtime.us-west-2.amazonaws.com"
+
@pytest.mark.asyncio
async def test_session_update_is_acked_with_session_updated(self, stub_aws_models):
handler = BedrockRealtime()
diff --git a/tests/unit/llms/bedrock/test_base_aws_llm.py b/tests/unit/llms/bedrock/test_base_aws_llm.py
index c755a7e6bc2..4350fdb699b 100644
--- a/tests/unit/llms/bedrock/test_base_aws_llm.py
+++ b/tests/unit/llms/bedrock/test_base_aws_llm.py
@@ -30,7 +30,7 @@ from litellm.llms.bedrock.base_aws_llm import (
BaseAWSLLM,
Boto3CredentialsInfo,
run_aws_signing,
- sign_request_off_loop_if_aws,
+ run_off_loop_if_aws,
)
from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
from litellm.llms.bedrock.common_utils import BedrockModelInfo
@@ -3443,7 +3443,7 @@ class TestGetRequestHeadersResign:
@pytest.mark.asyncio
-async def test_sign_request_off_loop_if_aws_keeps_the_loop_serving_while_credentials_refresh():
+async def test_run_off_loop_if_aws_keeps_the_loop_serving_while_credentials_refresh() -> None:
"""Regression for issue #40165: an AWS provider's signing (and the botocore credential refresh
inside it) must run off the event loop, so other requests keep being served meanwhile."""
probe = EventLoopProbe()
@@ -3456,7 +3456,7 @@ async def test_sign_request_off_loop_if_aws_keeps_the_loop_serving_while_credent
return dict(request.headers)
release = asyncio.create_task(probe.release_refresh_from_the_loop())
- signed = await sign_request_off_loop_if_aws(BaseAWSLLM(), sign, headers={"Content-Type": "application/json"})
+ signed = await run_off_loop_if_aws(BaseAWSLLM(), sign, headers={"Content-Type": "application/json"})
await release
assert "Authorization" in signed
diff --git a/tests/unit/llms/bedrock/test_claude_platform_provider.py b/tests/unit/llms/bedrock/test_claude_platform_provider.py
index 5d83cba5415..e43184df50e 100644
--- a/tests/unit/llms/bedrock/test_claude_platform_provider.py
+++ b/tests/unit/llms/bedrock/test_claude_platform_provider.py
@@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.router import GenericLiteLLMParams
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
WORKSPACE_ALIASES: Final = ("workspace_id", "aws_workspace_id", "anthropic_workspace_id", "anthropic-workspace-id")
@@ -683,6 +684,32 @@ async def test_anthropic_messages_claude_platform_sends_exact_body_through_stric
_assert_gateway_accepted(requests[0], auth_kwargs)
+@pytest.mark.asyncio
+async def test_anthropic_messages_claude_platform_workspace_and_region_lookups_leave_the_event_loop_free(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ import litellm
+
+ monkeypatch.setenv("ANTHROPIC_AWS_WORKSPACE_ID", "wrkspc_test")
+ monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+ vault: Final = host_in_loop_probing_vault(monkeypatch, "ANTHROPIC_AWS_WORKSPACE_ID", "AWS_REGION_NAME")
+ sigv4_without_region: Final = {key: value for key, value in SIGV4_KWARGS.items() if key != "aws_region_name"}
+ requests: list[dict] = []
+
+ response = await litellm.anthropic_messages(
+ model="bedrock/claude_platform/claude-sonnet-5-5",
+ messages=[{"role": "user", "content": "hello"}],
+ max_tokens=10,
+ client=_async_gateway_client(requests),
+ **sigv4_without_region,
+ )
+
+ assert vault.read.is_set()
+ assert not vault.stalled_the_loop.is_set()
+ assert response["content"][0]["text"] == "ok"
+ _assert_gateway_accepted(requests[0], sigv4_without_region)
+
+
def test_chat_completion_claude_platform_drops_context_management_and_gateway_accepts():
import litellm
diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py
index ee41af2110d..f252d9886d3 100644
--- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py
+++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py
@@ -20,7 +20,7 @@ from botocore.awsrequest import AWSRequest
import litellm
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig
-from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws
+from litellm.llms.bedrock.base_aws_llm import run_off_loop_if_aws
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.types.utils import LlmProviders
from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
@@ -1234,9 +1234,7 @@ async def test_mantle_signing_runs_off_the_event_loop():
return dict(request.headers)
release = asyncio.create_task(probe.release_refresh_from_the_loop())
- signed = await sign_request_off_loop_if_aws(
- BedrockMantleChatConfig(), sign, headers={"Content-Type": "application/json"}
- )
+ signed = await run_off_loop_if_aws(BedrockMantleChatConfig(), sign, headers={"Content-Type": "application/json"})
await release
assert "Authorization" in signed
diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py
index 8b46597b603..205c610a413 100644
--- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py
+++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py
@@ -5,13 +5,14 @@ import json
import logging
import threading
import time
-from collections.abc import Mapping
+from collections.abc import Awaitable, Callable, Mapping
from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
+import respx
from botocore.credentials import RefreshableCredentials
import litellm
@@ -66,6 +67,7 @@ from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIRe
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ImageObject, ImageResponse, ModelResponse, TranscriptionResponse
from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
_ACTIVE_KEY = "_code_interpreter_interception_active"
_SANDBOX_KEY = "_code_interpreter_interception_sandbox_key"
@@ -1245,6 +1247,155 @@ async def test_async_anthropic_messages_handler_signs_bedrock_off_the_event_loop
assert probe.served_during_refresh is True
+_BEDROCK_MESSAGE: Final = {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-5-5",
+ "content": [{"type": "text", "text": "hi"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 1, "output_tokens": 1},
+}
+_BEDROCK_RESPONSE: Final = {
+ "id": "resp_1",
+ "object": "response",
+ "created_at": 1,
+ "status": "completed",
+ "model": "openai.gpt-5.6-sol",
+ "output": [],
+ "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
+}
+_EMPTY_S3_LISTING: Final = (
+ b''
+ b''
+ b"batch-inputs0false"
+)
+_BATCH_LINE: Final = json.dumps(
+ {
+ "custom_id": "1",
+ "method": "POST",
+ "url": "/v1/chat/completions",
+ "body": {"model": "us.anthropic.claude-sonnet-5-5", "messages": [{"role": "user", "content": "hi"}]},
+ }
+).encode()
+_BATCH_FILE_ID: Final = "s3://batch-inputs/litellm-bedrock-files/batch.jsonl"
+_BEDROCK_ENTRY_POINTS: Final[tuple[tuple[str, Callable[[], Awaitable[object]], httpx.Response], ...]] = (
+ (
+ "anthropic_messages",
+ lambda: litellm.anthropic.messages.acreate(
+ model="bedrock/invoke/us.anthropic.claude-sonnet-5-5",
+ messages=[{"role": "user", "content": "hi"}],
+ max_tokens=16,
+ ),
+ httpx.Response(200, json=_BEDROCK_MESSAGE),
+ ),
+ (
+ "responses",
+ lambda: litellm.aresponses(model="bedrock/us.openai.gpt-5.6-sol", input="hi"),
+ httpx.Response(200, json=_BEDROCK_RESPONSE),
+ ),
+ (
+ "compact_responses",
+ lambda: litellm.acompact_responses(model="bedrock/us.openai.gpt-5.6-sol", input="hi"),
+ httpx.Response(200, json=_BEDROCK_RESPONSE),
+ ),
+ (
+ "search",
+ lambda: litellm.asearch(
+ query="hi",
+ search_provider="agentcore",
+ api_base="https://gw.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp",
+ ),
+ httpx.Response(200, json={"jsonrpc": "2.0", "id": 1, "result": {"content": []}}),
+ ),
+ (
+ "create_file",
+ lambda: litellm.acreate_file(
+ file=("batch.jsonl", _BATCH_LINE, "application/jsonl"),
+ purpose="batch",
+ custom_llm_provider="bedrock",
+ model="bedrock/us.anthropic.claude-sonnet-5-5",
+ ),
+ httpx.Response(200),
+ ),
+ (
+ "retrieve_file",
+ lambda: litellm.afile_retrieve(file_id=_BATCH_FILE_ID, custom_llm_provider="bedrock"),
+ httpx.Response(200, content=_BATCH_LINE),
+ ),
+ (
+ "delete_file",
+ lambda: litellm.afile_delete(file_id=_BATCH_FILE_ID, custom_llm_provider="bedrock"),
+ httpx.Response(204),
+ ),
+ (
+ "list_files",
+ lambda: litellm.afile_list(custom_llm_provider="bedrock"),
+ httpx.Response(200, content=_EMPTY_S3_LISTING),
+ ),
+ (
+ "file_content",
+ lambda: litellm.afile_content(file_id=_BATCH_FILE_ID, custom_llm_provider="bedrock"),
+ httpx.Response(200, content=_BATCH_LINE),
+ ),
+ (
+ "create_batch",
+ lambda: litellm.acreate_batch(
+ completion_window="24h",
+ endpoint="/v1/chat/completions",
+ input_file_id=_BATCH_FILE_ID,
+ custom_llm_provider="bedrock",
+ model="bedrock/us.anthropic.claude-sonnet-5-5",
+ aws_batch_role_arn="arn:aws:iam::123456789012:role/batch",
+ s3_output_bucket_name="batch-outputs",
+ ),
+ httpx.Response(200, json={"jobArn": "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/batch"}),
+ ),
+ (
+ "vector_store_search",
+ lambda: litellm.vector_stores.asearch(vector_store_id="KBID12345", query="hi", custom_llm_provider="bedrock"),
+ httpx.Response(200, json={"retrievalResults": []}),
+ ),
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("call", "upstream_answer"),
+ [pytest.param(call, answer, id=name) for name, call, answer in _BEDROCK_ENTRY_POINTS],
+)
+async def test_bedrock_secret_lookups_leave_the_event_loop_free(
+ call: Callable[[], Awaitable[object]],
+ upstream_answer: httpx.Response,
+ respx_mock: respx.MockRouter,
+ monkeypatch: pytest.MonkeyPatch,
+):
+ monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATESTTESTTESTTEST")
+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
+ monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+ monkeypatch.setenv("AWS_S3_BUCKET_NAME", "batch-inputs")
+ monkeypatch.setenv("AWS_S3_OUTPUT_BUCKET_NAME", "batch-outputs")
+ vault: Final = host_in_loop_probing_vault(
+ monkeypatch,
+ "AWS_REGION_NAME",
+ "AWS_BEDROCK_RUNTIME_ENDPOINT",
+ "AWS_BEARER_TOKEN_BEDROCK",
+ "AWS_S3_ENCRYPTION_KEY_ID",
+ "AWS_S3_BUCKET_OWNER",
+ "AGENTCORE_MCP_PROTOCOL_VERSION",
+ "AGENTCORE_SEARCH_TOOL_NAME",
+ "AGENTCORE_GATEWAY_TOKEN",
+ )
+ upstream: Final = respx_mock.route().mock(return_value=upstream_answer)
+
+ _ = await call()
+
+ assert vault.read.is_set()
+ assert not vault.stalled_the_loop.is_set()
+ assert ".us-west-2." in upstream.calls.last.request.url.host
+
+
@pytest.mark.asyncio
async def test_async_anthropic_messages_handler_passes_litellm_metadata():
"""Ensure litellm_metadata from kwargs is forwarded via update_from_kwargs.
diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py
index 3b33ba75eb3..75d713028d4 100644
--- a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py
+++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py
@@ -35,6 +35,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
)
from litellm.types.utils import CallTypes, ModelResponse
from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
@pytest.fixture
@@ -8063,3 +8064,44 @@ async def test_during_call_masking_rewrites_only_pii_block_in_content_list(
]
assert messages[1]["content"] == "Hello, how can I help you today?"
assert messages[2]["content"] == "who is the president of the united states?"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("guardrail_config", "upstream_answer"),
+ [
+ pytest.param(
+ {"guardrailIdentifier": "wf0hkdb5x07f", "guardrailVersion": "DRAFT"},
+ {"action": "NONE", "outputs": [], "assessments": []},
+ id="apply_guardrail",
+ ),
+ pytest.param(
+ {"checks": {"contentFilter": {"categories": [{"category": "VIOLENCE"}]}}, "content_filter_threshold": 0.5},
+ {"results": {"contentFilter": {"results": [{"category": "VIOLENCE", "severityScore": 0.1}]}}},
+ id="invoke_guardrail_checks",
+ ),
+ ],
+)
+async def test_guardrail_secret_lookups_leave_the_event_loop_free(
+ guardrail_config: dict[str, object],
+ upstream_answer: dict[str, object],
+ respx_mock: respx.MockRouter,
+ httpx_transport: None,
+ aws_test_credentials: None,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
+ monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+ vault: Final = host_in_loop_probing_vault(monkeypatch, "AWS_BEARER_TOKEN_BEDROCK", "AWS_REGION_NAME")
+ guardrail: Final = BedrockGuardrail(**guardrail_config)
+ upstream: Final = respx_mock.route().mock(return_value=httpx.Response(200, json=upstream_answer))
+
+ _ = await guardrail.make_bedrock_api_request(
+ source="INPUT",
+ messages=[{"role": "user", "content": "hello"}],
+ request_data={"messages": []},
+ )
+
+ assert vault.read.is_set()
+ assert not vault.stalled_the_loop.is_set()
+ assert upstream.calls.last.request.url.host == "bedrock-runtime.us-west-2.amazonaws.com"
diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
index 4dbe1f2ef8d..66c52a76318 100644
--- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
+++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
@@ -35,6 +35,7 @@ from litellm.proxy.utils import ProxyLogging
from litellm.passthrough.main import raise_for_provider_status
from litellm.llms.bedrock.common_utils import BedrockError
from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
+from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault
from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
BaseOpenAIPassThroughHandler,
@@ -2519,6 +2520,48 @@ class TestBedrockAgentRuntimePassthroughVirtualKeyLeak:
assert upstream["x-request-id"] == "trace-2"
+@pytest.mark.asyncio
+async def test_bedrock_agent_runtime_region_lookup_leaves_the_event_loop_free(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("litellm.proxy.proxy_server.master_key", SHARED_MASTER_KEY)
+ monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
+ monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+ litellm.in_memory_llm_clients_cache.flush_cache()
+ for ambient in TestBedrockAgentRuntimePassthroughVirtualKeyLeak.AMBIENT_AWS_ENV:
+ monkeypatch.delenv(ambient, raising=False)
+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "ak")
+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "sk")
+ monkeypatch.setenv("AWS_REGION_NAME", "us-west-2")
+ vault: Final = host_in_loop_probing_vault(monkeypatch, "AWS_REGION_NAME")
+
+ async def receive() -> dict[str, object]:
+ return {"type": "http.request", "body": b'{"retrievalQuery": {"text": "hi"}}', "more_body": False}
+
+ request: Final = Request(
+ {
+ "type": "http",
+ "method": "POST",
+ "path": "/bedrock/knowledgebases/KB1234567/retrieve",
+ "headers": [(b"content-type", b"application/json")],
+ "query_string": b"",
+ },
+ receive=receive,
+ )
+ with respx.mock(assert_all_called=True) as upstream:
+ retrieve: Final = upstream.post(
+ "https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases/KB1234567/retrieve"
+ ).mock(return_value=httpx.Response(200, json={"retrievalResults": []}))
+ _ = await bedrock_proxy_route(
+ endpoint="knowledgebases/KB1234567/retrieve",
+ request=request,
+ fastapi_response=Response(),
+ user_api_key_dict=UserAPIKeyAuth(api_key="sk-litellm-caller"),
+ )
+
+ assert vault.read.is_set()
+ assert not vault.stalled_the_loop.is_set()
+ assert retrieve.calls.last.request.headers["authorization"].startswith("AWS4-HMAC-SHA256")
+
+
class TestLLMPassthroughFactoryProxyRoute:
@pytest.mark.asyncio
async def test_llm_passthrough_factory_proxy_route_success(self):
From f925c067d2dc47d60e730965d0df6eeb38fc17d9 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 17:07:06 +0000
Subject: [PATCH 002/203] fix(tests): read owned proxy rows from the owned
proxy's database (#47333)
* fix(tests): read the stream stall cooldown spend rows from the owned proxy's database
The cooldown cells boot an owned proxy, which writes its spend logs to its
own scratch database since owned proxies went private, while the cells polled
the worker's database and never saw a row. Read each row through the gateway
that served the request, so the cells check the proxy they called.
* fix(tests): read passthrough spend rows from the owned proxy's database
The vertex regional uplift and passthrough stream pricing spend files boot
private owned proxies, then polled the worker database for spend rows that
only reach the proxy's own database. Read them through the owned proxy's
gateway instead.
Also hold the stream stall file's hold_headers peer past the longest row
wait, so a slow spend row cannot fail the peer teardown.
* fix(tests): check agent deletion on the owned proxy's database
The agent least privilege test deletes its agent through an owned proxy,
then checked the worker database for the row, which is always empty
there. Read it through the owned proxy's gateway so a delete that leaves
the row fails the test.
* fix(tests): read turn admission spend rows from the owned proxy's database
---
.../test_agent_least_privilege.py | 3 +-
.../test_hosted_vllm_stream_stall_wire.py | 2 +-
...test_responses_websocket_turn_admission.py | 11 +++---
..._passthrough_stream_pricing_paths_spend.py | 38 ++++++++++---------
...ertex_passthrough_regional_uplift_spend.py | 9 ++---
5 files changed, 31 insertions(+), 32 deletions(-)
diff --git a/tests/integration/authorization/test_agent_least_privilege.py b/tests/integration/authorization/test_agent_least_privilege.py
index 8a7f1596379..498a53c3ca9 100644
--- a/tests/integration/authorization/test_agent_least_privilege.py
+++ b/tests/integration/authorization/test_agent_least_privilege.py
@@ -8,7 +8,6 @@ import httpx
import pytest
import yaml
from integration._support.client import Gateway, eventually, object_value, string_value
-from integration._support.database import read_rows
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, Wire, wire_server
@@ -108,7 +107,7 @@ def _create_agent(candidate: Gateway, wire: Wire, agent_name: str, cleanups: Exi
def _delete_agent(candidate: Gateway, agent_id: str) -> None:
deleted: Final = candidate.request("DELETE", f"/v1/agents/{agent_id}")
assert deleted.status_code == 200, deleted.text
- assert read_rows('SELECT agent_id FROM "LiteLLM_AgentsTable" WHERE agent_id=%s', (agent_id,)) == []
+ assert candidate.rows('SELECT agent_id FROM "LiteLLM_AgentsTable" WHERE agent_id=%s', (agent_id,)) == []
def _card(candidate: Gateway, agent_id: str, key: str) -> httpx.Response:
diff --git a/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py b/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py
index 00e5388b474..fad7898871d 100644
--- a/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py
+++ b/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py
@@ -42,8 +42,8 @@ _RESPONSES_TARGET: Final = "/v1/responses"
_STREAM_TIMEOUT_SECONDS: Final = 1
_DEPLOYMENT_TIMEOUT_SECONDS: Final = 30
_PAUSE_SECONDS: Final = 3.0
-_HOLD_SECONDS: Final = 30.0
_ROW_SECONDS: Final = 60
+_HOLD_SECONDS: Final = 2.0 * _ROW_SECONDS
_EXTRA: Final[dict[str, JsonValue]] = {"num_retries": 0, "cache": {"no-cache": True}}
_ROWS: Final = 'SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE model_group=%s'
_EITHER_GROUP_ROWS: Final = 'SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE model_group IN (%s, %s)'
diff --git a/tests/integration/routing/test_responses_websocket_turn_admission.py b/tests/integration/routing/test_responses_websocket_turn_admission.py
index a246c89e7b3..87b56a92979 100644
--- a/tests/integration/routing/test_responses_websocket_turn_admission.py
+++ b/tests/integration/routing/test_responses_websocket_turn_admission.py
@@ -27,7 +27,6 @@ from integration._support.client import (
object_value,
string_value,
)
-from integration._support.database import read_rows
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.responses_ws_peer import (
PROVIDER_MODEL,
@@ -260,9 +259,9 @@ async def _until(read: Callable[[], float], satisfied: Callable[[float], bool])
return await asyncio.to_thread(eventually, read, satisfied, 15)
-def _failure_rows(key: str) -> list[dict[str, JsonValue]]:
+def _failure_rows(gateway: Gateway, key: str) -> list[dict[str, JsonValue]]:
rows: Final = eventually(
- lambda: read_rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
+ lambda: gateway.rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
lambda found: any(row["status"] == "failure" for row in found),
seconds=70,
)
@@ -291,7 +290,7 @@ def test_a_turn_past_the_deployment_budget_is_refused_on_the_open_socket_and_log
_assert_refused(second)
_assert_refused(third)
assert deployment.forwarded_turns() == 1
- failures: Final = _failure_rows(key)
+ failures: Final = _failure_rows(limited, key)
assert len(failures) == 1, failures
assert failures[0]["model_group"] == deployment.name, failures
assert tag in list_value(failures[0]["request_tags"]), failures
@@ -355,7 +354,7 @@ def test_a_turn_whose_admission_check_fails_gets_a_500_frame_and_the_socket_serv
_assert_turn_check_failed(third, fault)
completed(fourth)
assert deployment.forwarded_turns() == 2
- failures: Final = _failure_rows(key)
+ failures: Final = _failure_rows(faulting, key)
assert len(failures) == 1, failures
assert failures[0]["model_group"] == deployment.name, failures
recorded: Final = object_value(object_value(failures[0]["metadata"])["error_information"])
@@ -387,7 +386,7 @@ def test_a_turn_still_streaming_when_the_next_turns_check_fails_completes_and_is
billed: Final = completed(streamed)
assert deployment.forwarded_turns() == 1
rows: Final = eventually(
- lambda: read_rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
+ lambda: faulting.rows(SPEND_ROWS_FOR_KEY, (_key_hash(key),)),
lambda found: any(row["request_id"] == billed["id"] for row in found),
seconds=70,
)
diff --git a/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py b/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py
index f091a8f8376..0ead544a2ee 100644
--- a/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py
+++ b/tests/integration/spend/test_passthrough_stream_pricing_paths_spend.py
@@ -12,7 +12,6 @@ import httpx
import pytest
import yaml
from integration._support.client import Gateway, eventually, gateway_from_environment
-from integration._support.database import read_rows
from integration._support.forward_proxy import tunnelling_forward_proxy
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.tls import server_context, write_self_signed_cert
@@ -289,9 +288,9 @@ def gateway_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
yield _Rig(proxy, upstream)
-def _spend_row(key: str) -> dict[str, JsonValue]:
+def _spend_row(gateway: Gateway, key: str) -> dict[str, JsonValue]:
rows: Final = eventually(
- lambda: read_rows(
+ lambda: gateway.rows(
'SELECT spend, custom_llm_provider FROM "LiteLLM_SpendLogs" WHERE api_key=%s',
(sha256(key.encode()).hexdigest(),),
),
@@ -301,14 +300,14 @@ def _spend_row(key: str) -> dict[str, JsonValue]:
return rows[0]
-def _spend_for_key(key: str) -> float:
- spend: Final = _spend_row(key)["spend"]
+def _spend_for_key(gateway: Gateway, key: str) -> float:
+ spend: Final = _spend_row(gateway, key)["spend"]
assert isinstance(spend, float), key
return spend
-def _provider_for_key(key: str) -> JsonValue:
- return _spend_row(key)["custom_llm_provider"]
+def _provider_for_key(gateway: Gateway, key: str) -> JsonValue:
+ return _spend_row(gateway, key)["custom_llm_provider"]
def _post_and_read(rig: _Rig, path: str, body: Mapping[str, JsonValue], headers: Mapping[str, str]) -> None:
@@ -347,7 +346,7 @@ def test_claude_on_vertex_passthrough_spend_carries_the_regional_uplift_whether_
{**body, "stream": action == "streamRawPredict"},
{"x-litellm-api-key": f"Bearer {key}", "Authorization": f"Bearer {_CALLER_GOOGLE_TOKEN}"},
)
- spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
+ spend: Final = {cell: _spend_for_key(rig.proxy, key) for cell, key in keys.items()}
standard: Final = _list_price(f"vertex_ai/{_CLAUDE_ROUTE_MODEL}")
regional: Final = standard * _uplift(f"vertex_ai/{_CLAUDE_ROUTE_MODEL}")
assert spend == pytest.approx(
@@ -370,7 +369,7 @@ def test_v1beta_generate_content_on_a_vertex_deployment_bills_the_deployment_loc
keys: Final = {cell: scenario.key() for cell in cells}
for (deployment, action), key in keys.items():
_post_and_read(rig, f"/v1beta/models/{deployment}:{action}", body, {"x-goog-api-key": key})
- spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
+ spend: Final = {cell: _spend_for_key(rig.proxy, key) for cell, key in keys.items()}
standard: Final = _list_price(f"vertex_ai/{_GEMINI}")
regional: Final = standard * _uplift(f"vertex_ai/{_GEMINI}")
assert spend == pytest.approx(
@@ -404,8 +403,8 @@ def test_gemini_passthrough_bills_a_named_custom_priced_deployment_only_for_keys
for key in (allowed[action], plain[action]):
_post_and_read(rig, f"/gemini/v1beta/models/{_GEMINI}:{action}", body, {"x-goog-api-key": key})
spend: Final = {
- **{("allowed", action): _spend_for_key(key) for action, key in allowed.items()},
- **{("plain", action): _spend_for_key(key) for action, key in plain.items()},
+ **{("allowed", action): _spend_for_key(rig.proxy, key) for action, key in allowed.items()},
+ **{("plain", action): _spend_for_key(rig.proxy, key) for action, key in plain.items()},
}
custom_price: Final = _PROMPT_TOKENS * _CUSTOM_INPUT_RATE + _COMPLETION_TOKENS * _CUSTOM_OUTPUT_RATE
list_price: Final = _list_price(f"gemini/{_GEMINI}")
@@ -441,7 +440,10 @@ def test_openai_passthrough_stream_cut_off_after_its_usage_bills_what_the_finish
assert response.status_code == 200
with pytest.raises(httpx.RemoteProtocolError):
response.read()
- spend: Final = {"finished": _spend_for_key(finished_key), "cut off": _spend_for_key(cut_key)}
+ spend: Final = {
+ "finished": _spend_for_key(rig.proxy, finished_key),
+ "cut off": _spend_for_key(rig.proxy, cut_key),
+ }
list_price: Final = _list_price(_OPENAI_MODEL)
assert spend == pytest.approx({"finished": list_price, "cut off": list_price})
@@ -484,7 +486,7 @@ def test_a_body_base_model_reprices_passthrough_spend_only_for_keys_that_allow_c
},
_caller_headers(route, key),
)
- spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
+ spend: Final = {cell: _spend_for_key(rig.proxy, key) for cell, key in keys.items()}
forwarded: Final = rig.upstream.drain()
marked_bodies: Final = {
cell: tuple(request.body for request in forwarded if marker.encode() in request.body)
@@ -521,7 +523,7 @@ def test_a_non_string_body_base_model_bills_the_list_price_instead_of_nothing(ri
{"contents": [{"role": "user", "parts": [{"text": uuid.uuid4().hex}]}], "base_model": 123},
_caller_headers(route, key),
)
- spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
+ spend: Final = {cell: _spend_for_key(rig.proxy, key) for cell, key in keys.items()}
list_price: Final = {
"vertex_ai": _list_price(f"vertex_ai/{_GEMINI}"),
"gemini": _list_price(f"gemini/{_GEMINI}"),
@@ -538,8 +540,8 @@ def test_gemini_passthrough_through_a_gemini_api_base_gateway_logs_and_bills_as_
keys: Final = {action: scenario.key() for action in _GEMINI_ACTIONS}
for action, key in keys.items():
_post_and_read(gateway_rig, f"/gemini/v1beta/models/{_GEMINI}:{action}", body, {"x-goog-api-key": key})
- providers: Final = {action: _provider_for_key(key) for action, key in keys.items()}
- spend: Final = {action: _spend_for_key(key) for action, key in keys.items()}
+ providers: Final = {action: _provider_for_key(gateway_rig.proxy, key) for action, key in keys.items()}
+ spend: Final = {action: _spend_for_key(gateway_rig.proxy, key) for action, key in keys.items()}
forwarded: Final = tuple(
path
for path in (urlsplit(request.target).path for request in gateway_rig.upstream.drain())
@@ -570,8 +572,8 @@ def test_a_configured_passthrough_to_a_non_google_gateway_bills_as_vertex_whatev
body,
{"Authorization": f"Bearer {key}"},
)
- providers: Final = {cell: _provider_for_key(key) for cell, key in keys.items()}
- spend: Final = {cell: _spend_for_key(key) for cell, key in keys.items()}
+ providers: Final = {cell: _provider_for_key(gateway_rig.proxy, key) for cell, key in keys.items()}
+ spend: Final = {cell: _spend_for_key(gateway_rig.proxy, key) for cell, key in keys.items()}
short_path_spend: Final = {action: spend[("short gateway path", action)] for action in _GEMINI_ACTIONS}
assert providers == {cell: "vertex_ai" for cell in cells}
assert {action: spend[("vertex resource", action)] for action in _GEMINI_ACTIONS} == pytest.approx(
diff --git a/tests/integration/spend/test_vertex_passthrough_regional_uplift_spend.py b/tests/integration/spend/test_vertex_passthrough_regional_uplift_spend.py
index eaa8f52d4f4..0b2dd8910d6 100644
--- a/tests/integration/spend/test_vertex_passthrough_regional_uplift_spend.py
+++ b/tests/integration/spend/test_vertex_passthrough_regional_uplift_spend.py
@@ -10,7 +10,6 @@ import httpx
import pytest
import yaml
from integration._support.client import Gateway, eventually, gateway_from_environment
-from integration._support.database import read_rows
from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.tls import server_context, write_self_signed_cert
@@ -150,12 +149,12 @@ def _spend(rig: _Rig, key: str, location: str, action: str, model: str = _MODEL)
body: Final = response.read()
assert response.status_code == 200, body
call_id: Final = response.headers["x-litellm-call-id"]
- return _logged_spend(call_id)
+ return _logged_spend(rig.proxy, call_id)
-def _logged_spend(call_id: str) -> float:
+def _logged_spend(gateway: Gateway, call_id: str) -> float:
rows: Final = eventually(
- lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)),
+ lambda: gateway.rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)),
lambda found: len(found) == 1,
seconds=70,
)
@@ -193,7 +192,7 @@ def test_custom_priced_vertex_passthrough_bills_the_deployment_price_even_when_t
call_id: Final = response.headers["x-litellm-call-id"]
with pytest.raises(httpx.RemoteProtocolError):
response.read()
- cut_off: Final = _logged_spend(call_id)
+ cut_off: Final = _logged_spend(rig.proxy, call_id)
custom_price: Final = _PROMPT_TOKENS * _CUSTOM_INPUT_RATE + _COMPLETION_TOKENS * _CUSTOM_OUTPUT_RATE
assert {**finished, "cut off": cut_off} == pytest.approx(
{_GENERATE: custom_price, _STREAM: custom_price, "cut off": custom_price}
From b55024658cd6b1db66284cec95ab8af7689762c7 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
<158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 20 Oct 2026 17:09:53 +0000
Subject: [PATCH 003/203] fix(proxy)!: stop update_cache from resurrecting
evicted auth cache entries (#46997)
* fix(proxy): stop update_cache from resurrecting evicted auth cache entries
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test(proxy): update update_cache pins for synchronous write-back
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test(proxy): wrap long assert in update_cache tag test
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test(proxy): cover refill race and pin TTL with injected clock
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* fix(proxy): count global proxy spend for every request with a user, cached on this worker or not
* fix(proxy): add global proxy spend with an atomic Redis increment that never re-creates a reset counter
The global proxy spend bump read the total, awaited, and wrote old + cost back
from a background task, so a budget reset that deleted the key in between got
the pre-reset total written back, and workers overwrote each other's totals.
RedisCache.async_increment_if_present runs INCRBYFLOAT only when the key
exists, keeping its TTL, and this worker's copy is then set from the shared
total with no await between its read and write. The bump now runs whenever a
proxy max_budget is set, matching the DB writer, and the armed post-call read
and the batch-read declaration it used are removed.
* fix(proxy): report the global spend Redis increment to service metrics and request traces
The global proxy spend increment now emits Redis service success and
failure events under the spend_counters target and passes the request
span through, so it shows as redis.incr spend_counters like the write
it replaced.
The unit tests run the real RedisCache increment against fakeredis
instead of mocks, and a new test checks the span lands under the
request span on success and on a Redis failure.
* test(proxy): keep the spend counter Redis fake off the network
* test(proxy): count increment_if_present as a shared-cache call in the key-family guard
* fix(proxy): keep the newer global spend total when Redis replies arrive out of order
---------
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
---
litellm/caching/dual_cache.py | 26 --
litellm/caching/redis_cache.py | 47 ++-
litellm/integrations/otel/README.md | 2 +-
litellm/integrations/otel/model/spans.py | 1 +
.../proxy/hooks/proxy_track_cost_callback.py | 13 -
litellm/proxy/proxy_server.py | 343 +++------------
.../spend/test_global_proxy_spend_counter.py | 58 +++
.../test_request_redis_batch_post_call.py | 143 +------
.../integrations/otel/test_otel_v2_logger.py | 1 +
.../proxy/proxy_server/test_spend_counters.py | 398 ++++++++++++++++--
.../test_spend_counter_batch.py | 19 -
...test_proxy_server_endpoints_and_startup.py | 270 ++----------
tests/unit/test_internal_context.py | 4 +-
13 files changed, 594 insertions(+), 731 deletions(-)
create mode 100644 tests/integration/spend/test_global_proxy_spend_counter.py
diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py
index bef04a5c23c..e6214a3650c 100644
--- a/litellm/caching/dual_cache.py
+++ b/litellm/caching/dual_cache.py
@@ -59,16 +59,6 @@ class PendingBatchRead:
previous_access_times: dict[str, float | None]
-@dataclass(frozen=True, slots=True)
-class DeclaredBatchRead:
- """A ``async_batch_get_cache`` split in two: the memory half done, the Redis half declared on a ``RedisBatch``
- so it rides that batch's next round trip, resolved later with ``async_resolve_batch_get``."""
-
- keys: tuple[str, ...]
- pending: PendingBatchRead
- result: BatchResult[Mapping[str, object]] | None
-
-
def _log_deferred_increment_failure(future: asyncio.Future[float]) -> None:
if future.cancelled():
return
@@ -382,22 +372,6 @@ class DualCache(BaseCache):
await self.in_memory_cache.async_set_cache(key, value, **self._backfill_kwargs(kwargs))
return merged
- async def declare_batch_get(self, keys: Sequence[str], batch: RedisBatch) -> DeclaredBatchRead:
- pending: Final = await self._prepare_batch_get(
- list(keys),
- local_only=False,
- throttle_redis=False,
- )
- return DeclaredBatchRead(
- keys=tuple(keys),
- pending=pending,
- result=batch.mget(pending.redis_keys) if pending.redis_keys else None,
- )
-
- async def async_resolve_batch_get(self, declared: DeclaredBatchRead) -> list[object | None]:
- redis_result: Final = None if declared.result is None else await declared.result
- return await self._apply_batch_get(declared.pending, redis_result)
-
async def async_batch_get_cache(
self,
keys: list,
diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py
index 461ac5d947f..eb17facfb12 100644
--- a/litellm/caching/redis_cache.py
+++ b/litellm/caching/redis_cache.py
@@ -134,7 +134,12 @@ _INCREMENT_WITH_FLOOR_LUA: Final = (
"return count"
)
+_INCREMENT_IF_PRESENT_LUA: Final = (
+ "if redis.call('EXISTS', KEYS[1]) == 0 then return false end return redis.call('INCRBYFLOAT', KEYS[1], ARGV[1])"
+)
+
_LUA_COUNT: Final = TypeAdapter(int)
+_LUA_OPTIONAL_TOTAL: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
_OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...])
@@ -675,7 +680,7 @@ class RedisCache(BaseCache):
# redis namespaces
self.namespace = namespace
# for high traffic, we store the redis results in memory and then batch write to redis
- self.redis_batch_writing_buffer: list = []
+ self.redis_batch_writing_buffer: list[tuple[str, object]] = []
if redis_flush_size is None:
self.redis_flush_size: int = 100
else:
@@ -1568,6 +1573,46 @@ class RedisCache(BaseCache):
count: Final = await _redis_client.eval(_INCREMENT_WITH_FLOOR_LUA, 1, namespaced_key, value, ttl)
return _LUA_COUNT.validate_python(count)
+ @_redis_circuit_breaker_guard
+ async def async_increment_if_present(
+ self, key: str, value: float, parent_otel_span: Span | None = None
+ ) -> float | None:
+ _redis_client: Final = self._async_commands()
+ namespaced_key: Final = self.check_and_fix_namespace(key=key)
+ start_time: Final = time.time()
+ try:
+ total: Final = _LUA_OPTIONAL_TOTAL.validate_python(
+ await _redis_client.eval(_INCREMENT_IF_PRESENT_LUA, 1, namespaced_key, value)
+ )
+ except Exception as e:
+ failed_at: Final = time.time()
+ asyncio.create_task(
+ self.service_logger_obj.async_service_failure_hook(
+ service=ServiceTypes.REDIS,
+ duration=failed_at - start_time,
+ error=e,
+ call_type="async_increment_if_present",
+ caller=_get_call_stack_info(),
+ start_time=start_time,
+ end_time=failed_at,
+ parent_otel_span=parent_otel_span,
+ )
+ )
+ raise
+ end_time: Final = time.time()
+ asyncio.create_task(
+ self.service_logger_obj.async_service_success_hook(
+ service=ServiceTypes.REDIS,
+ duration=end_time - start_time,
+ call_type="async_increment_if_present",
+ caller=_get_call_stack_info(),
+ start_time=start_time,
+ end_time=end_time,
+ parent_otel_span=parent_otel_span,
+ )
+ )
+ return total
+
async def flush_cache_buffer(self):
print_verbose(f"flushing to redis....reached size of buffer {len(self.redis_batch_writing_buffer)}")
await self.async_set_cache_pipeline(self.redis_batch_writing_buffer)
diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md
index 7414a451038..06c2da117b5 100644
--- a/litellm/integrations/otel/README.md
+++ b/litellm/integrations/otel/README.md
@@ -91,7 +91,7 @@ way (`pod_lock`, `budget_reset`, `spend_queue`, `health_check`, `scheduler_queue
rather than a bare `redis.set`; `tests/unit/test_internal_context.py` scans every
module under `litellm/` and `enterprise/` that calls a shared cache or declares a
read or write on the request batch (`reserve_redis_batch_reads`,
-`declare_batch_get`, `batch.mget`, `batch.set`, `batch.script`) and fails when
+`batch.mget`, `batch.set`, `batch.script`) and fails when
one has no declared family, with the process-local `InMemoryCache` callers listed
as the only exemptions. A batch op carries the family that was active when it was
declared, so the routing prefetch armed before deployment selection
diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py
index 76fc969cfc0..d8be607150c 100644
--- a/litellm/integrations/otel/model/spans.py
+++ b/litellm/integrations/otel/model/spans.py
@@ -214,6 +214,7 @@ _SERVICE_VERB_BY_CALL_TYPE: Final[dict[str, str]] = {
"increment_cache": "incr",
"async_increment": "incr",
"async_increment_pipeline": "incr",
+ "async_increment_if_present": "incr",
"delete_cache": "delete",
"async_delete_cache": "delete",
"async_rpush": "rpush",
diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py
index 4434fe57ab6..38a76028ddc 100644
--- a/litellm/proxy/hooks/proxy_track_cost_callback.py
+++ b/litellm/proxy/hooks/proxy_track_cost_callback.py
@@ -359,7 +359,6 @@ class _ProxyDBLogger(CustomLogger):
increment_spend_counters,
proxy_logging_obj,
update_cache,
- update_cache_read_keys,
)
verbose_proxy_logger.debug("INSIDE _PROXY_track_cost_callback")
@@ -454,12 +453,6 @@ class _ProxyDBLogger(CustomLogger):
request_tags=tags,
model_access_groups=model_access_groups,
project_id=project_id,
- update_cache_read_keys=update_cache_read_keys(
- user_id=user_id,
- end_user_id=end_user_id,
- tags=tags,
- response_cost=response_cost,
- ),
)
if not charged:
return
@@ -783,7 +776,6 @@ async def _update_database_and_spend_counters(
request_tags: list[str] | None = None,
model_access_groups: Sequence[str] | None = None,
project_id: str | None = None,
- update_cache_read_keys: Sequence[str] = (),
) -> bool:
"""The reservation is reconciled before the spend is persisted, from its own read. One spend counter batch then
spans the database write and the counter update, so the post-call counters are read with a single MGET after the
@@ -825,7 +817,6 @@ async def _update_database_and_spend_counters(
request_tags=request_tags,
model_access_groups=model_access_groups,
project_id=project_id,
- update_cache_read_keys=update_cache_read_keys,
)
@@ -846,10 +837,7 @@ async def _update_database_and_spend_counters_in_batch(
request_tags: list[str] | None,
model_access_groups: Sequence[str] | None,
project_id: str | None,
- update_cache_read_keys: Sequence[str],
) -> bool:
- from litellm.proxy.proxy_server import arm_update_cache_read
-
try:
charged: Final = await proxy_logging_obj.db_spend_update_writer.update_database(
token=user_api_key,
@@ -881,7 +869,6 @@ async def _update_database_and_spend_counters_in_batch(
await _release_budget_reservation(budget_reservation=budget_reservation)
return False
- await arm_update_cache_read(update_cache_read_keys)
try:
await increment_spend_counters(
token=user_api_key,
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 2c6d2e2736a..0af6036efa7 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -146,7 +146,6 @@ from litellm.proxy._types import (
TransformRequestBody,
UserAPIKeyAuth,
)
-from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
from litellm.proxy.common_utils.callback_utils import (
is_sensitive_callback_key,
normalize_callback_names,
@@ -308,10 +307,8 @@ from litellm._logging import (
verbose_router_logger,
)
from litellm.caching.caching import DualCache, RedisCache
-from litellm.caching.dual_cache import DeclaredBatchRead
from litellm.caching.redis_batch import (
active_post_call_redis_batch,
- active_request_redis_batches,
drain_post_call_redis_batches,
)
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError, is_redis_timeout_failure
@@ -4236,73 +4233,61 @@ async def run_spend_counter_pipeline(pending: Sequence[PendingSpendIncrement]) -
return tuple(float(current_value) for current_value in results or ())
-def update_cache_read_keys(
- user_id: str | None,
- end_user_id: str | None,
- tags: Sequence[object] | None,
- response_cost: float | None,
-) -> tuple[str, ...]:
- if response_cost is None:
- return ()
- user_keys: tuple[str, ...] = (user_id, GLOBAL_PROXY_SPEND_CACHE_KEY) if user_id is not None else ()
- end_user_keys: tuple[str, ...] = (end_user_cache_key(end_user_id),) if end_user_id is not None else ()
- tag_keys: tuple[str, ...] = tuple(tag_cache_key(tag) for tag in tags or () if isinstance(tag, str) and tag)
- return user_keys + end_user_keys + tag_keys
+_CachedSpendObjectT = TypeVar(
+ "_CachedSpendObjectT", LiteLLM_UserTable, LiteLLM_EndUserTable, LiteLLM_TeamTableCachedObj, LiteLLM_TagTable
+)
-_UPDATE_CACHE_PREFETCH_SLOT: Final = "update_cache_read"
-
-
-async def arm_update_cache_read(keys: Sequence[str], cache: DualCache | None = None) -> None:
- """Declares the ``update_cache`` read on the request pipeline once the spend is persisted, so it rides the same
- round trip as the post-call spend counter read instead of its own."""
- request: Final = active_request_redis_batches()
- target: Final = user_api_key_cache if cache is None else cache
- if request is None or target.redis_cache is None or not keys:
+def _add_spend_to_locally_cached_object(
+ cache: UserApiKeyCache, key: str, model_type: type[_CachedSpendObjectT], response_cost: float
+) -> None:
+ current: Final = cache.get_cache(key=key, local_only=True, model_type=model_type)
+ if current is None:
return
- with service_target(AUTH_OBJECTS_TARGET):
- request.prefetched[_UPDATE_CACHE_PREFETCH_SLOT] = await target.declare_batch_get(
- keys, request.batch(target.redis_cache)
- )
+ cache.set_cache(
+ key=key,
+ value=current.model_copy(update={"spend": (current.spend or 0.0) + response_cost}),
+ local_only=True,
+ model_type=model_type,
+ ttl=get_management_object_ttl(cache),
+ )
-async def _take_armed_update_cache_read(keys: Sequence[str], cache: DualCache) -> Mapping[str, object] | None:
- request: Final = active_request_redis_batches()
- if request is None:
+async def _add_spend_to_shared_global_proxy_spend(
+ cache: UserApiKeyCache, response_cost: float, parent_otel_span: Span | None
+) -> float | None:
+ if cache.redis_cache is None:
return None
- armed: Final = request.prefetched.pop(_UPDATE_CACHE_PREFETCH_SLOT, None)
- if not isinstance(armed, DeclaredBatchRead) or armed.keys != tuple(keys):
- return None
- values: Final = await cache.async_resolve_batch_get(armed)
- return MappingProxyType({key: value for key, value in zip(keys, values) if value is not None})
-
-
-async def _read_update_cache_values(
- keys: Sequence[str], parent_otel_span: Span | None, cache: DualCache | None = None
-) -> Mapping[str, object]:
- """One batched read for every object ``update_cache`` refreshes; a failed read leaves them all untouched,
- exactly as a failed per-object GET left that object untouched."""
- if not keys:
- return MappingProxyType({})
- target: Final = user_api_key_cache if cache is None else cache
try:
- armed: Final = await _take_armed_update_cache_read(keys, target)
- if armed is not None:
- return armed
- values: Final = await target.async_batch_get_cache(
- keys=list(keys), parent_otel_span=parent_otel_span, throttle_redis=False
+ return await cache.redis_cache.async_increment_if_present(
+ key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=response_cost, parent_otel_span=parent_otel_span
)
except Exception as e:
verbose_proxy_logger.warning(
- "Spend tracking - failed to read cached spend objects. Budget enforcement may use stale spend values. "
- "keys=%s - %s",
- keys,
+ "Spend tracking - failed to add spend to the global proxy spend in Redis. "
+ "Budget enforcement may use stale spend values. response_cost=%s - %s",
+ response_cost,
str(e),
)
- return MappingProxyType({})
- if values is None:
- return MappingProxyType({})
- return MappingProxyType({key: value for key, value in zip(keys, values) if value is not None})
+ return None
+
+
+async def _add_spend_to_global_proxy_spend(
+ cache: UserApiKeyCache, response_cost: float, parent_otel_span: Span | None
+) -> None:
+ local_before: Final = cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, local_only=True)
+ shared_total: Final = await _add_spend_to_shared_global_proxy_spend(cache, response_cost, parent_otel_span)
+ local_total: Final = cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, local_only=True)
+ if not isinstance(local_total, (int, float)):
+ return
+ if shared_total is not None and local_total != local_before and shared_total < local_total:
+ return
+ cache.set_cache(
+ key=GLOBAL_PROXY_SPEND_CACHE_KEY,
+ value=local_total + response_cost if shared_total is None else shared_total,
+ local_only=True,
+ ttl=get_management_object_ttl(cache),
+ )
async def update_cache(
@@ -4313,6 +4298,7 @@ async def update_cache(
response_cost: float | None,
parent_otel_span: Span | None,
tags: list[str] | None = None,
+ cache: UserApiKeyCache | None = None,
):
"""
Use this to update the cache with new user spend.
@@ -4320,14 +4306,7 @@ async def update_cache(
Put any alerting logic in here.
"""
- values_to_update_in_cache: Final[list[tuple[str, object]]] = []
- with service_target(AUTH_OBJECTS_TARGET):
- cached_values: Final = await _read_update_cache_values(
- keys=update_cache_read_keys(
- user_id=user_id, end_user_id=end_user_id, tags=tags, response_cost=response_cost
- ),
- parent_otel_span=parent_otel_span,
- )
+ target: Final = user_api_key_cache if cache is None else cache
### UPDATE KEY SPEND ###
async def _update_key_cache(token: str, response_cost: float):
@@ -4337,7 +4316,7 @@ async def update_cache(
else:
hashed_token = token
verbose_proxy_logger.debug("_update_key_cache: hashed_token=%s", hashed_token)
- existing_spend_obj = await user_api_key_cache.async_get_cache(key=hashed_token, model_type=UserAPIKeyAuth)
+ existing_spend_obj = await target.async_get_cache(key=hashed_token, model_type=UserAPIKeyAuth)
verbose_proxy_logger.debug("_update_key_cache: existing_spend_obj=%s", existing_spend_obj)
if existing_spend_obj is None:
return
@@ -4379,184 +4358,17 @@ async def update_cache(
)
# set cooldown on alert
- ### UPDATE USER SPEND ###
- async def _update_user_cache():
- ## UPDATE CACHE FOR USER ID + GLOBAL PROXY
- if response_cost is None:
- return
- user_ids: Final = [user_id]
+ def _add_spend(key: str, model_type: type[_CachedSpendObjectT], cost: float) -> None:
try:
- for _id in user_ids:
- # Fetch the existing cost for the given user
- if _id is None:
- continue
- cached_user = cached_values.get(_id)
- if cached_user is None:
- # do nothing if there is no cache value
- return
- existing_spend_obj = CacheCodec.deserialize(cached_user, LiteLLM_UserTable)
- if existing_spend_obj is None:
- return
- verbose_proxy_logger.debug(
- "_update_user_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost
- )
-
- existing_spend = existing_spend_obj.spend or 0.0
- # Calculate the new cost by adding the existing cost and response_cost
- new_spend = existing_spend + response_cost
-
- existing_spend_obj.spend = new_spend
- values_to_update_in_cache.append(
- (
- _id,
- CacheCodec.serialize(existing_spend_obj, model_type=LiteLLM_UserTable),
- )
- )
- ## UPDATE GLOBAL PROXY ##
- global_proxy_spend: Final = cached_values.get(GLOBAL_PROXY_SPEND_CACHE_KEY)
- if not isinstance(global_proxy_spend, (int, float)):
- # do nothing if not in cache
- return
- elif response_cost is not None:
- increment: Final = global_proxy_spend + response_cost
- values_to_update_in_cache.append((GLOBAL_PROXY_SPEND_CACHE_KEY, increment))
+ _add_spend_to_locally_cached_object(target, key, model_type, cost)
except Exception as e:
verbose_proxy_logger.warning(
- "Spend tracking - failed to update user spend in cache. "
+ "Spend tracking - failed to update %s spend in cache. "
"Budget enforcement may use stale spend values. "
- "user_id=%s, response_cost=%s - %s\n%s",
- user_id,
- response_cost,
- str(e),
- traceback.format_exc(),
- )
-
- ### UPDATE END-USER SPEND ###
- async def _update_end_user_cache():
- if end_user_id is None or response_cost is None:
- return
-
- _id: Final = end_user_cache_key(end_user_id)
- try:
- # Fetch the existing cost for the given user
- cached_end_user: Final = cached_values.get(_id)
- if cached_end_user is None:
- # if user does not exist in LiteLLM_UserTable, create a new user
- # do nothing if end-user not in api key cache
- return
- existing_spend_obj: Final = CacheCodec.deserialize(cached_end_user, LiteLLM_EndUserTable)
- if existing_spend_obj is None:
- return
- verbose_proxy_logger.debug(
- "_update_end_user_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost
- )
-
- existing_spend: Final = existing_spend_obj.spend or 0.0
- # Calculate the new cost by adding the existing cost and response_cost
- new_spend: Final = existing_spend + response_cost
-
- existing_spend_obj.spend = new_spend
- values_to_update_in_cache.append(
- (
- _id,
- CacheCodec.serialize(existing_spend_obj, model_type=LiteLLM_EndUserTable),
- )
- )
- except Exception as e:
- verbose_proxy_logger.warning(
- "Spend tracking - failed to update end user spend in cache. "
- "Budget enforcement may use stale spend values. "
- "end_user_id=%s, response_cost=%s - %s\n%s",
- end_user_id,
- response_cost,
- str(e),
- traceback.format_exc(),
- )
-
- ### UPDATE TEAM SPEND ###
- def _update_team_cache():
- if team_id is None or response_cost is None:
- return
-
- _id: Final = f"team_id:{team_id}"
- try:
- current_team: Final = CacheCodec.deserialize(
- user_api_key_cache.get_cache(key=_id, local_only=True), LiteLLM_TeamTableCachedObj
- )
- if current_team is None:
- return
- verbose_proxy_logger.debug(
- "_update_team_db: existing spend: %s; response_cost: %s", current_team, response_cost
- )
- user_api_key_cache.set_cache(
- key=_id,
- value=CacheCodec.serialize(
- current_team.model_copy(update={"spend": (current_team.spend or 0.0) + response_cost}),
- model_type=LiteLLM_TeamTableCachedObj,
- ),
- local_only=True,
- ttl=get_management_object_ttl(user_api_key_cache),
- )
- except Exception as e:
- verbose_proxy_logger.warning(
- "Spend tracking - failed to update team spend in cache. "
- "Budget enforcement may use stale spend values. "
- "team_id=%s, response_cost=%s - %s\n%s",
- team_id,
- response_cost,
- str(e),
- traceback.format_exc(),
- )
-
- ### UPDATE TAG SPEND ###
- async def _update_tag_cache():
- """
- Update the tag cache with the new spend.
- """
- if tags is None or response_cost is None:
- return
-
- try:
- for tag_name in tags:
- if not tag_name or not isinstance(tag_name, str):
- continue
-
- cache_key = tag_cache_key(tag_name)
- # Fetch the existing tag object from cache
- cached_tag = cached_values.get(cache_key)
- if cached_tag is None:
- # do nothing if tag not in api key cache
- continue
-
- existing_tag_obj = CacheCodec.deserialize(cached_tag, LiteLLM_TagTable)
- if existing_tag_obj is None:
- continue
-
- verbose_proxy_logger.debug(
- "_update_tag_cache: existing spend for tag=%s: %s; response_cost: %s",
- tag_name,
- existing_tag_obj,
- response_cost,
- )
-
- existing_spend = existing_tag_obj.spend or 0.0
- # Calculate the new cost by adding the existing cost and response_cost
- new_spend = existing_spend + response_cost
-
- existing_tag_obj.spend = new_spend
- values_to_update_in_cache.append(
- (
- cache_key,
- CacheCodec.serialize(existing_tag_obj, model_type=LiteLLM_TagTable),
- )
- )
- except Exception as e:
- verbose_proxy_logger.warning(
- "Spend tracking - failed to update tag spend in cache. "
- "Budget enforcement may use stale spend values. "
- "tags=%s, response_cost=%s - %s\n%s",
- tags,
- response_cost,
+ "key=%s, response_cost=%s - %s\n%s",
+ model_type.__name__,
+ key,
+ cost,
str(e),
traceback.format_exc(),
)
@@ -4564,42 +4376,23 @@ async def update_cache(
with service_target(AUTH_OBJECTS_TARGET):
if token is not None and response_cost is not None:
await _update_key_cache(token=token, response_cost=response_cost)
+ if response_cost is None:
+ return
- if user_id is not None:
- await _update_user_cache()
+ if user_id is not None:
+ _add_spend(user_id, LiteLLM_UserTable, response_cost)
+ if end_user_id is not None:
+ _add_spend(end_user_cache_key(end_user_id), LiteLLM_EndUserTable, response_cost)
+ if team_id is not None:
+ _add_spend(f"team_id:{team_id}", LiteLLM_TeamTableCachedObj, response_cost)
+ for tag_name in tags or ():
+ if tag_name and isinstance(tag_name, str):
+ _add_spend(tag_cache_key(tag_name), LiteLLM_TagTable, response_cost)
- if end_user_id is not None:
- await _update_end_user_cache()
-
- if team_id is not None:
- _update_team_cache()
-
- if tags is not None:
- await _update_tag_cache()
-
- global_proxy_spend_key: Final = GLOBAL_PROXY_SPEND_CACHE_KEY
- local_object_updates: Final = tuple((k, v) for k, v in values_to_update_in_cache if k != global_proxy_spend_key)
- shared_scalar_updates: Final = tuple((k, v) for k, v in values_to_update_in_cache if k == global_proxy_spend_key)
-
- if local_object_updates:
- with service_target(AUTH_OBJECTS_TARGET):
- asyncio.create_task(
- user_api_key_cache.async_set_cache_pipeline(
- cache_list=list(local_object_updates),
- ttl=get_management_object_ttl(user_api_key_cache),
- litellm_parent_otel_span=parent_otel_span,
- local_only=True,
- )
- )
- if shared_scalar_updates:
- with service_target(SPEND_COUNTERS_TARGET):
- asyncio.create_task(
- user_api_key_cache.async_set_cache_pipeline(
- cache_list=list(shared_scalar_updates),
- ttl=get_management_object_ttl(user_api_key_cache),
- litellm_parent_otel_span=parent_otel_span,
- )
- )
+ if litellm.max_budget <= 0:
+ return
+ with service_target(SPEND_COUNTERS_TARGET):
+ await _add_spend_to_global_proxy_spend(target, response_cost, parent_otel_span)
def run_ollama_serve():
diff --git a/tests/integration/spend/test_global_proxy_spend_counter.py b/tests/integration/spend/test_global_proxy_spend_counter.py
new file mode 100644
index 00000000000..0185e48ffbd
--- /dev/null
+++ b/tests/integration/spend/test_global_proxy_spend_counter.py
@@ -0,0 +1,58 @@
+import os
+import uuid
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.caching.redis_cache import RedisCache
+from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
+from litellm.proxy.proxy_server import update_cache
+
+
+async def _add_request_cost(cache: UserApiKeyCache, response_cost: float) -> None:
+ await update_cache(
+ token=None,
+ user_id=None,
+ end_user_id=None,
+ team_id=None,
+ response_cost=response_cost,
+ parent_otel_span=None,
+ cache=cache,
+ )
+
+
+@pytest.mark.asyncio
+async def test_global_proxy_spend_adds_every_instance_cost_and_stays_cleared_after_a_budget_reset(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(litellm, "max_budget", 100.0)
+ redis_cache: Final = RedisCache(
+ host=os.environ["REDIS_HOST"],
+ port=int(os.environ["REDIS_PORT"]),
+ namespace=f"global-proxy-spend-{uuid.uuid4().hex}",
+ )
+ gateway: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache)
+ peer: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache)
+
+ try:
+ await gateway.async_set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0, ttl=60)
+ await _add_request_cost(peer, 0.5)
+ await _add_request_cost(gateway, 0.25)
+ before_reset: Final = (
+ gateway.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, local_only=True),
+ await redis_cache.async_get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY),
+ )
+ ttl_before_reset: Final = await redis_cache.async_get_ttl(key=GLOBAL_PROXY_SPEND_CACHE_KEY)
+
+ await gateway.async_delete_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY)
+ await _add_request_cost(peer, 0.5)
+ await _add_request_cost(gateway, 0.25)
+
+ assert before_reset == (10.75, 10.75)
+ assert ttl_before_reset is not None and 0 < ttl_before_reset <= 60
+ assert await redis_cache.async_get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY) is None
+ finally:
+ await redis_cache.async_delete_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY)
diff --git a/tests/unit/caching/test_request_redis_batch_post_call.py b/tests/unit/caching/test_request_redis_batch_post_call.py
index 5de785e44db..4b73bb6dd34 100644
--- a/tests/unit/caching/test_request_redis_batch_post_call.py
+++ b/tests/unit/caching/test_request_redis_batch_post_call.py
@@ -10,31 +10,27 @@ import datetime
import hashlib
import json
from collections.abc import Awaitable, Callable, Mapping, Sequence
-from unittest.mock import AsyncMock, MagicMock
import pytest
import litellm
-from litellm._internal_context import current_service_target
from litellm.caching.caching import Cache
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_batch import (
active_post_call_redis_batch,
- active_request_redis_batches,
drain_post_call_redis_batches,
flush_post_call_redis_batches,
request_redis_batch_scope,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
-from litellm.proxy.auth.auth_object_prefetch import AUTH_OBJECTS_TARGET
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
PARALLEL_RELEASE_SCRIPT,
TOKEN_INCREMENT_SCRIPT,
ParallelSlotAcquisition,
- RequestRateLimiterStash,
PROXY_MaxParallelRequestsHandler_v3,
+ RequestRateLimiterStash,
)
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement
from litellm.proxy.utils import InternalUsageCache
@@ -518,140 +514,3 @@ async def test_a_cancelled_post_call_flush_keeps_the_shared_spend_counter_and_co
assert redis_cache.alone == [], "a cancel says nothing about the shared counter, so Redis keeps it"
assert spend_cache.in_memory_cache.get_cache("spend:key:k1") == 3.5, "the local copy counts the cancelled spend"
assert spend_cache.in_memory_cache.get_cache("spend:team:t1") is None, "an absent local copy is not seeded"
-
-
-@pytest.mark.asyncio
-async def test_the_update_cache_read_armed_before_accounting_rides_the_pipeline_of_the_reconcile_read():
- from litellm.proxy.proxy_server import _read_update_cache_values, arm_update_cache_read
-
- client = FakeClient(_ok_replies)
- redis_cache = PostCallFakeRedisCache(client)
- cache = DualCache()
- cache.attach_redis_cache(redis_cache)
- keys = ["user-1", "team_id:t1"]
-
- with request_redis_batch_scope() as request:
- await arm_update_cache_read(keys, cache=cache)
- assert client.pipelines == []
- await request.batch(redis_cache).mget(["spend:key:k1"]) # the spend reconcile read of the same request
- values = await _read_update_cache_values(keys, None, cache=cache)
-
- assert len(client.pipelines) == 1
- assert client.pipelines[0].commands == [("MGET", "user-1", "team_id:t1"), ("MGET", "spend:key:k1")]
- assert values == {"user-1": {"spend": 1.0}, "team_id:t1": {"spend": 1.0}}
- assert redis_cache.alone == []
- assert active_request_redis_batches() is None
-
-
-@pytest.mark.asyncio
-async def test_the_armed_update_cache_read_is_declared_under_the_auth_objects_family():
- """The user, team and tag rows the accounting reads are auth objects, so the pipeline that carries
- the armed read renders ``redis.pipeline auth_objects``, not a bare ``redis.pipeline``."""
- from litellm.proxy.proxy_server import _read_update_cache_values, arm_update_cache_read
-
- client = FakeClient(_ok_replies)
- redis_cache = PostCallFakeRedisCache(client)
- cache = DualCache()
- cache.attach_redis_cache(redis_cache)
- pipeline_targets: list[str | None] = [] # mutable-ok: filled by the recording hook
-
- async def record(**kwargs: object) -> None:
- pipeline_targets.append(current_service_target())
-
- redis_cache.service_logger_obj.async_service_success_hook = record # pyright: ignore[reportAttributeAccessIssue] # fake, records the hook call
-
- with request_redis_batch_scope():
- await arm_update_cache_read(["user-1", "team_id:t1"], cache=cache)
- await _read_update_cache_values(["user-1", "team_id:t1"], None, cache=cache)
- await asyncio.gather(*(t for t in asyncio.all_tasks() if t is not asyncio.current_task()))
-
- assert pipeline_targets == [AUTH_OBJECTS_TARGET]
-
-
-@pytest.mark.asyncio
-async def test_an_update_cache_read_armed_for_other_keys_is_ignored_and_the_read_happens_as_before():
- from litellm.proxy.proxy_server import _read_update_cache_values, arm_update_cache_read
-
- redis_cache = PostCallFakeRedisCache(FakeClient(_ok_replies))
- redis_cache.store["team_id:t1"] = {"spend": 2.0}
- cache = DualCache()
- cache.attach_redis_cache(redis_cache)
-
- with request_redis_batch_scope():
- await arm_update_cache_read(["user-1"], cache=cache)
- values = await _read_update_cache_values(["team_id:t1"], None, cache=cache)
-
- assert values == {"team_id:t1": {"spend": 2.0}}
- assert ("MGET", ("team_id:t1",)) in redis_cache.alone
-
-
-@pytest.mark.asyncio
-async def test_the_update_cache_read_sees_a_cached_spend_written_while_the_spend_was_persisted(monkeypatch):
- from litellm.proxy import proxy_server
- from litellm.proxy.hooks.proxy_track_cost_callback import _update_database_and_spend_counters
-
- cached_user_spend = {"user-1": 1.0}
-
- def replies(command: tuple[object, ...]) -> object:
- if command[0] == "MGET":
- return [
- json.dumps({"spend": cached_user_spend[key]}) if key in cached_user_spend else b"0.5"
- for key in command[1:]
- ]
- return _ok_replies(command)
-
- client = FakeClient(replies)
- redis_cache = PostCallFakeRedisCache(client)
- spend_cache = DualCache()
- spend_cache.attach_redis_cache(redis_cache)
- user_cache = DualCache()
- user_cache.attach_redis_cache(redis_cache)
- monkeypatch.setattr(proxy_server, "spend_counter_cache", spend_cache)
- monkeypatch.setattr(proxy_server, "user_api_key_cache", user_cache)
-
- async def _read_on_the_request_pipeline_then_a_concurrent_callback_writes_the_user(**kwargs: object) -> bool:
- request = active_request_redis_batches()
- assert request is not None
- await request.batch(redis_cache).mget(["key-object"])
- cached_user_spend["user-1"] = 5.0
- return True
-
- proxy_logging_obj = MagicMock()
- proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock(
- side_effect=_read_on_the_request_pipeline_then_a_concurrent_callback_writes_the_user
- )
- reservation = {
- "reserved_cost": 0.5,
- "entries": [
- {
- "counter_key": "spend:key:k1",
- "entity_type": "Key",
- "entity_id": "k1",
- "reserved_cost": 0.5,
- "applied_adjustment": 0.0,
- }
- ],
- "finalized": False,
- }
-
- with request_redis_batch_scope():
- charged = await _update_database_and_spend_counters(
- proxy_logging_obj=proxy_logging_obj,
- increment_spend_counters=proxy_server.increment_spend_counters,
- user_api_key="k1",
- user_id="user-1",
- end_user_id=None,
- team_id=None,
- org_id=None,
- kwargs={},
- completion_response=None,
- start_time=datetime.datetime.now(),
- end_time=datetime.datetime.now(),
- response_cost=0.2,
- budget_reservation=reservation,
- update_cache_read_keys=("user-1",),
- )
- values = await proxy_server._read_update_cache_values(("user-1",), None)
-
- assert charged is True
- assert values == {"user-1": {"spend": 5.0}}, client.pipelines
diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py
index 10e12981ec1..1f9d3af71d9 100644
--- a/tests/unit/integrations/otel/test_otel_v2_logger.py
+++ b/tests/unit/integrations/otel/test_otel_v2_logger.py
@@ -1928,6 +1928,7 @@ def test_service_span_is_named_by_purpose_when_the_producer_declares_a_target():
("async_batch_get_cache", "redis.mget auth_objects", "redis.mget"),
("async_set_cache_pipeline_with_ttls", "redis.set auth_objects", "redis.set"),
("async_increment_pipeline", "redis.incr auth_objects", "redis.incr"),
+ ("async_increment_if_present", "redis.incr auth_objects", "redis.incr"),
("async_delete_cache", "redis.delete auth_objects", "redis.delete"),
("async_scan_iter", "redis.scan auth_objects", "redis.scan"),
("request_redis_batch", "redis.pipeline auth_objects", "redis.pipeline"),
diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py
index 3ff4293ba93..d273ef8f42f 100644
--- a/tests/unit/proxy/proxy_server/test_spend_counters.py
+++ b/tests/unit/proxy/proxy_server/test_spend_counters.py
@@ -24,15 +24,31 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from datetime import datetime
-from typing import Final, cast
-from unittest.mock import AsyncMock, MagicMock
+from typing import Final, Literal, cast
+from unittest.mock import AsyncMock, MagicMock, call
import pytest
+from fakeredis import FakeAsyncRedis, FakeServer
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import StatusCode
+import litellm
import litellm.proxy.proxy_server as ps
+from litellm._service_logger import ServiceLogging
+from litellm._internal_context import current_service_target
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker
+from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY
+from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
+from litellm.models.end_user import LiteLLM_EndUserTable
+from litellm.models.tag import LiteLLM_TagTable
from litellm.models.team import LiteLLM_TeamTable, LiteLLM_TeamTableCachedObj
+from litellm.models.user import LiteLLM_UserTable
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_local
-from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
+from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_user_cache_key, tag_cache_key
from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team
from .conftest import normalize
@@ -1559,31 +1575,6 @@ async def test_invalidate_spend_counter_swallows_redis_failure_no_raise(monkeypa
# ---------------------------------------------------------------------------
-@pytest.mark.asyncio
-async def test_update_cache_no_cached_entities_schedules_pipeline_flush(monkeypatch):
- fake_user_cache = _make_user_api_key_cache(get_value=None)
- monkeypatch.setattr(ps, "user_api_key_cache", fake_user_cache)
-
- await ps.update_cache(
- token=None,
- user_id="u1",
- end_user_id="eu1",
- team_id="t1",
- response_cost=1.0,
- parent_otel_span=None,
- tags=["x"],
- )
-
- assert fake_user_cache.async_get_cache.await_count == 0
- fake_user_cache.async_batch_get_cache.assert_awaited_once()
- assert fake_user_cache.async_batch_get_cache.await_args.kwargs["keys"] == [
- "u1",
- f"{ps.litellm_proxy_admin_name}:spend",
- "end_user_id:eu1",
- "tag:x",
- ]
-
-
@pytest.mark.asyncio
async def test_update_cache_user_cache_failure_invalid_state_is_swallowed(monkeypatch):
"""An inner _update_user_cache raising must not propagate — update_cache
@@ -1671,3 +1662,354 @@ async def test_update_cache_never_restores_a_team_a_management_write_replaced(
seen_by_auth: Final = cache.get_cache(key=f"team_id:{team_id}", model_type=LiteLLM_TeamTableCachedObj)
assert (seen_by_auth.blocked if seen_by_auth is not None else None) == expected_blocked
+
+
+_CachedSpendObject = LiteLLM_UserTable | LiteLLM_EndUserTable | LiteLLM_TeamTableCachedObj | LiteLLM_TagTable
+
+
+def _cached_spend_object(
+ kind: Literal["user", "end_user", "team", "tag"], models: list[str]
+) -> tuple[str, _CachedSpendObject]:
+ match kind:
+ case "user":
+ return "race-user", LiteLLM_UserTable(user_id="race-user", models=models)
+ case "end_user":
+ return end_user_cache_key("race-end-user"), LiteLLM_EndUserTable(
+ user_id="race-end-user", blocked=False, models=models
+ )
+ case "team":
+ return "team_id:race-team", LiteLLM_TeamTableCachedObj(team_id="race-team", models=models)
+ case "tag":
+ return tag_cache_key("race-tag"), LiteLLM_TagTable(tag_name="race-tag", models=models)
+
+
+class _RunsOnKeyRead(InMemoryCache):
+ def __init__(self, on_read: Callable[[], Awaitable[None]]) -> None:
+ super().__init__()
+ self._on_read: Final = on_read
+
+ async def async_get_cache(self, key: str, **kwargs: object) -> object:
+ await self._on_read()
+ return await super().async_get_cache(key, **kwargs)
+
+
+def _cache_that_runs_on_key_read(on_read: Callable[[], Awaitable[None]]) -> UserApiKeyCache:
+ return UserApiKeyCache(in_memory_cache=InMemoryCache(), key_object_in_memory_cache=_RunsOnKeyRead(on_read))
+
+
+@pytest.fixture
+def proxy_max_budget(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(litellm, "max_budget", 100.0)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["user", "end_user", "team", "tag"])
+@pytest.mark.parametrize(
+ ("management_write", "expected"),
+ [("none", (["gpt-a"], 0.25)), ("evict", None), ("refresh", (["gpt-b"], 0.25))],
+)
+async def test_update_cache_adds_spend_to_the_entry_a_management_write_left_mid_flight(
+ kind: Literal["user", "end_user", "team", "tag"],
+ management_write: Literal["none", "evict", "refresh"],
+ expected: tuple[list[str], float] | None,
+) -> None:
+ key, seeded = _cached_spend_object(kind, models=["gpt-a"])
+ _, refreshed = _cached_spend_object(kind, models=["gpt-b"])
+
+ async def _write() -> None:
+ if management_write == "none":
+ return
+ evict_local(cache_keys=(key,), user_api_key_cache=cache)
+ if management_write == "refresh":
+ cache.set_cache(key=key, value=refreshed, local_only=True)
+
+ cache: Final = _cache_that_runs_on_key_read(on_read=_write)
+ cache.set_cache(key=key, value=seeded, local_only=True)
+
+ await ps.update_cache(
+ token="sk-race",
+ user_id="race-user",
+ end_user_id="race-end-user",
+ team_id="race-team",
+ response_cost=0.25,
+ parent_otel_span=None,
+ tags=["race-tag"],
+ cache=cache,
+ )
+ await _drain_spend_cache_writes()
+
+ stored: Final = cache.get_cache(key=key, local_only=True, model_type=type(seeded))
+ assert ((stored.models, stored.spend) if stored is not None else None) == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+@pytest.mark.parametrize("user", ["cached_on_this_worker", "not_cached_on_this_worker", "none"])
+async def test_update_cache_adds_spend_to_the_global_proxy_spend_for_every_request(
+ user: Literal["cached_on_this_worker", "not_cached_on_this_worker", "none"],
+) -> None:
+ cache: Final = UserApiKeyCache()
+ cache.set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0)
+ if user == "cached_on_this_worker":
+ cache.set_cache(key="spender", value=LiteLLM_UserTable(user_id="spender", spend=1.0))
+
+ await ps.update_cache(
+ token=None,
+ user_id=None if user == "none" else "spender",
+ end_user_id=None,
+ team_id=None,
+ response_cost=0.25,
+ parent_otel_span=None,
+ cache=cache,
+ )
+ await _drain_spend_cache_writes()
+
+ assert cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY) == 10.25
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+async def test_update_cache_never_brings_back_a_global_proxy_spend_a_budget_reset_cleared() -> None:
+ async def _budget_reset() -> None:
+ await cache.async_delete_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY)
+
+ cache: Final = _cache_that_runs_on_key_read(on_read=_budget_reset)
+ cache.set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0)
+
+ await ps.update_cache(
+ token="sk-spender",
+ user_id="spender",
+ end_user_id=None,
+ team_id=None,
+ response_cost=0.25,
+ parent_otel_span=None,
+ cache=cache,
+ )
+ await _drain_spend_cache_writes()
+
+ assert cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY) is None
+
+
+class _FakeRedisRunningIncrementIfPresent(FakeAsyncRedis):
+ def __init__(self, server: FakeServer, after_increment: Callable[[], object] = lambda: None) -> None:
+ super().__init__(server=server)
+ self._after_increment: Final = after_increment
+
+ async def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> bytes | None:
+ key, increment = keys_and_args
+ if not await self.exists(key):
+ return None
+ total: Final = await self.incrbyfloat(key, increment)
+ self._after_increment()
+ return str(total).encode()
+
+
+class _RedisCacheOnClient(RedisCache):
+ def __init__(self, client: FakeAsyncRedis) -> None:
+ self._injected_client: Final = client
+ self.namespace = None
+ self._circuit_breaker = RedisCircuitBreaker(failure_threshold=5, recovery_timeout=30)
+ self.service_logger_obj = ServiceLogging()
+ self.default_ttl = None
+
+ def init_async_client(self) -> FakeAsyncRedis:
+ return self._injected_client
+
+
+class _FakeRedisHoldingOneReply(FakeAsyncRedis):
+ def __init__(self, server: FakeServer, held_increment: float, applied: asyncio.Event, released: asyncio.Event) -> None:
+ super().__init__(server=server)
+ self._held_increment: Final = held_increment
+ self._applied: Final = applied
+ self._released: Final = released
+
+ async def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> bytes | None:
+ key, increment = keys_and_args
+ total: Final = await self.incrbyfloat(key, increment)
+ if float(increment) == self._held_increment:
+ self._applied.set()
+ await self._released.wait()
+ return str(total).encode()
+
+
+_SharedGlobalSpendOutcome = Literal["peers_added_spend", "reset_elsewhere", "reset_while_redis_answers", "redis_down"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+@pytest.mark.parametrize(
+ ("outcome", "shared_total", "expected"),
+ [
+ ("peers_added_spend", 11.75, 12.0),
+ ("reset_elsewhere", None, 10.25),
+ ("reset_while_redis_answers", 9.75, None),
+ ("redis_down", 10.0, 10.25),
+ ],
+)
+async def test_update_cache_moves_this_workers_global_proxy_spend_to_the_shared_total(
+ outcome: _SharedGlobalSpendOutcome, shared_total: float | None, expected: float | None
+) -> None:
+ def _reset_this_worker() -> None:
+ if outcome == "reset_while_redis_answers":
+ cache.in_memory_cache.delete_cache(GLOBAL_PROXY_SPEND_CACHE_KEY)
+
+ server: Final = FakeServer()
+ redis: Final = _FakeRedisRunningIncrementIfPresent(server, after_increment=_reset_this_worker)
+ if shared_total is not None:
+ await redis.set(GLOBAL_PROXY_SPEND_CACHE_KEY, shared_total)
+ server.connected = outcome != "redis_down"
+ cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnClient(redis))
+ cache.set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0, local_only=True)
+
+ await ps.update_cache(
+ token=None,
+ user_id=None,
+ end_user_id=None,
+ team_id=None,
+ response_cost=0.25,
+ parent_otel_span=None,
+ cache=cache,
+ )
+
+ assert cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, local_only=True) == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+async def test_update_cache_keeps_the_newest_global_proxy_spend_when_redis_replies_out_of_order() -> None:
+ held_increment_applied: Final = asyncio.Event()
+ held_reply_released: Final = asyncio.Event()
+ server: Final = FakeServer()
+ redis: Final = _FakeRedisHoldingOneReply(
+ server, held_increment=0.25, applied=held_increment_applied, released=held_reply_released
+ )
+ await redis.set(GLOBAL_PROXY_SPEND_CACHE_KEY, 10.0)
+ cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnClient(redis))
+ cache.set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0, local_only=True)
+
+ async def _request(response_cost: float) -> None:
+ await ps.update_cache(
+ token=None,
+ user_id=None,
+ end_user_id=None,
+ team_id=None,
+ response_cost=response_cost,
+ parent_otel_span=None,
+ cache=cache,
+ )
+
+ earlier: Final = asyncio.create_task(_request(0.25))
+ await held_increment_applied.wait()
+ await _request(0.5)
+ held_reply_released.set()
+ await earlier
+
+ assert cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, local_only=True) == 10.75
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+async def test_update_cache_adds_the_global_spend_scalar_in_redis_as_spend_counters() -> None:
+ target_during_increment: Final = MagicMock()
+ redis: Final = _FakeRedisRunningIncrementIfPresent(
+ FakeServer(), after_increment=lambda: target_during_increment(current_service_target())
+ )
+ await redis.set(GLOBAL_PROXY_SPEND_CACHE_KEY, 10.0)
+ cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnClient(redis))
+
+ await ps.update_cache(
+ token=None,
+ user_id="u1",
+ end_user_id=None,
+ team_id=None,
+ response_cost=2.0,
+ parent_otel_span=None,
+ cache=cache,
+ )
+
+ assert target_during_increment.call_args_list == [call("spend_counters")]
+ assert float(await redis.get(GLOBAL_PROXY_SPEND_CACHE_KEY)) == 12.0
+ assert current_service_target() is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+@pytest.mark.parametrize(("redis_up", "expected_status"), [(True, StatusCode.OK), (False, StatusCode.ERROR)])
+async def test_update_cache_traces_the_global_spend_redis_call_under_the_request_span(
+ monkeypatch: pytest.MonkeyPatch, redis_up: bool, expected_status: StatusCode
+) -> None:
+ exporter: Final = InMemorySpanExporter()
+ provider: Final = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ monkeypatch.setattr(
+ litellm,
+ "service_callback",
+ [OpenTelemetry(config=OpenTelemetryConfig(exporter=exporter), tracer_provider=provider)],
+ )
+ server: Final = FakeServer()
+ redis: Final = _FakeRedisRunningIncrementIfPresent(server)
+ await redis.set(GLOBAL_PROXY_SPEND_CACHE_KEY, 10.0)
+ server.connected = redis_up
+ cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnClient(redis))
+ request_span: Final = provider.get_tracer("test").start_span("request")
+
+ await ps.update_cache(
+ token=None,
+ user_id=None,
+ end_user_id=None,
+ team_id=None,
+ response_cost=2.0,
+ parent_otel_span=request_span,
+ cache=cache,
+ )
+ await _drain_spend_cache_writes()
+ request_span.end()
+
+ redis_spans: Final = tuple(
+ (span.attributes.get("call_type"), span.status.status_code, span.parent.span_id)
+ for span in exporter.get_finished_spans()
+ if span.name == "redis" and span.attributes is not None and span.parent is not None
+ )
+ assert redis_spans == (
+ ("async_increment_if_present", expected_status, request_span.get_span_context().span_id),
+ )
+
+
+class _UnwritableKeyUserApiKeyCache(UserApiKeyCache):
+ def __init__(self, unwritable_key: str) -> None:
+ super().__init__(in_memory_cache=InMemoryCache())
+ self._unwritable_key: Final = unwritable_key
+
+ def set_cache(self, key: str | None, value: object, local_only: bool = False, **kwargs: object):
+ if key == self._unwritable_key:
+ raise RuntimeError("cache write failed")
+ return super().set_cache(key, value, local_only=local_only, **kwargs)
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("proxy_max_budget")
+async def test_update_cache_still_adds_spend_to_the_others_when_one_cached_object_fails() -> None:
+ cache: Final = _UnwritableKeyUserApiKeyCache(unwritable_key="broken-user")
+ cache.in_memory_cache.set_cache(
+ key="broken-user",
+ value=CacheCodec.serialize(LiteLLM_UserTable(user_id="broken-user", spend=1.0), model_type=LiteLLM_UserTable),
+ )
+ cache.set_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY, value=10.0)
+ cache.set_cache(key="team_id:still-billed", value=LiteLLM_TeamTableCachedObj(team_id="still-billed", spend=1.0))
+
+ await ps.update_cache(
+ token=None,
+ user_id="broken-user",
+ end_user_id=None,
+ team_id="still-billed",
+ response_cost=0.25,
+ parent_otel_span=None,
+ cache=cache,
+ )
+ await _drain_spend_cache_writes()
+
+ team: Final = cache.get_cache(key="team_id:still-billed", model_type=LiteLLM_TeamTableCachedObj)
+ assert (team.spend if team is not None else None, cache.get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY)) == (
+ 1.25,
+ 10.25,
+ )
diff --git a/tests/unit/proxy/spend_tracking/test_spend_counter_batch.py b/tests/unit/proxy/spend_tracking/test_spend_counter_batch.py
index 1fddfaaa766..8ba79a45ff3 100644
--- a/tests/unit/proxy/spend_tracking/test_spend_counter_batch.py
+++ b/tests/unit/proxy/spend_tracking/test_spend_counter_batch.py
@@ -737,22 +737,3 @@ async def test_a_resized_reservation_is_carried_at_its_resized_cost_to_the_count
assert reservation["reserved_cost"] == pytest.approx(0.1)
assert redis.store["spend:key:hashed"] == pytest.approx(9.9)
assert redis.store["spend:team:team"] == pytest.approx(2.1)
-
-
-@pytest.mark.asyncio
-async def test_update_cache_reads_an_object_redis_gained_right_after_a_batch_read_missed_it(monkeypatch):
- """DualCache throttles repeated batch reads of a key that just missed; the per-object GET update_cache used to
- issue never did, so its batched read must not either."""
- from litellm.caching.dual_cache import DualCache
-
- redis = CountingRedis()
- cache = DualCache(redis_cache=redis)
- monkeypatch.setattr(ps, "user_api_key_cache", cache)
- assert await cache.async_batch_get_cache(keys=["team_id:team"]) == [None]
- redis.store["team_id:team"] = {"spend": 1.0}
- assert await cache.async_batch_get_cache(keys=["team_id:team"]) == [None]
-
- assert await ps._read_update_cache_values(keys=["team_id:team"], parent_otel_span=None) == {
- "team_id:team": {"spend": 1.0}
- }
- assert redis.commands.count("MGET team_id:team") == 2
diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py
index 7eb3f557971..14196d4a1ac 100644
--- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py
+++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py
@@ -6189,53 +6189,24 @@ async def test_tag_cache_update_called():
"""
Test that update_cache updates tag cache when tags are provided.
"""
- from litellm.caching.caching import DualCache
- from litellm.proxy.proxy_server import user_api_key_cache
+ from litellm.caching.in_memory_cache import InMemoryCache
+ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
- cache = DualCache()
+ cache = UserApiKeyCache(in_memory_cache=InMemoryCache())
+ cache.in_memory_cache.set_cache("tag:test-tag", {"tag_name": "test-tag", "spend": 10.0})
- setattr(
- litellm.proxy.proxy_server,
- "user_api_key_cache",
- cache,
+ await litellm.proxy.proxy_server.update_cache(
+ token=None,
+ user_id=None,
+ end_user_id=None,
+ team_id=None,
+ response_cost=5.0,
+ parent_otel_span=None,
+ tags=["test-tag"],
+ cache=cache,
)
- mock_tag_obj = {
- "tag_name": "test-tag",
- "spend": 10.0,
- }
-
- set_cache_called: Final = asyncio.Event()
-
- async def _record_set_cache(*args: object, **kwargs: object) -> None:
- set_cache_called.set()
-
- with patch.object(cache, "async_batch_get_cache", new=AsyncMock(return_value=[mock_tag_obj])) as mock_get_cache:
- with patch.object(
- cache, "async_set_cache_pipeline", new=AsyncMock(side_effect=_record_set_cache)
- ) as mock_set_cache:
- await litellm.proxy.proxy_server.update_cache(
- token=None,
- user_id=None,
- end_user_id=None,
- team_id=None,
- response_cost=5.0,
- parent_otel_span=None,
- tags=["test-tag"],
- )
-
- assert await asyncio.wait_for(set_cache_called.wait(), timeout=5)
-
- mock_get_cache.assert_awaited_once_with(keys=["tag:test-tag"], parent_otel_span=None, throttle_redis=False)
- mock_set_cache.assert_awaited_once()
-
- call_args = mock_set_cache.call_args
- cache_list = call_args.kwargs["cache_list"]
-
- assert len(cache_list) == 1
- cache_key, cache_value = cache_list[0]
- assert cache_key == "tag:test-tag"
- assert cache_value["spend"] == 15.0
+ assert cache.in_memory_cache.get_cache("tag:test-tag")["spend"] == 15.0
@pytest.mark.asyncio
@@ -6243,59 +6214,26 @@ async def test_tag_cache_update_multiple_tags():
"""
Test that multiple tags are updated in cache.
"""
- from litellm.caching.caching import DualCache
- from litellm.proxy.proxy_server import user_api_key_cache
+ from litellm.caching.in_memory_cache import InMemoryCache
+ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
- cache = DualCache()
+ cache = UserApiKeyCache(in_memory_cache=InMemoryCache())
+ cache.in_memory_cache.set_cache("tag:tag1", {"tag_name": "tag1", "spend": 10.0})
+ cache.in_memory_cache.set_cache("tag:tag2", {"tag_name": "tag2", "spend": 20.0})
- setattr(
- litellm.proxy.proxy_server,
- "user_api_key_cache",
- cache,
+ await litellm.proxy.proxy_server.update_cache(
+ token=None,
+ user_id=None,
+ end_user_id=None,
+ team_id=None,
+ response_cost=5.0,
+ parent_otel_span=None,
+ tags=["tag1", "tag2"],
+ cache=cache,
)
- mock_tag1_obj = {"tag_name": "tag1", "spend": 10.0}
- mock_tag2_obj = {"tag_name": "tag2", "spend": 20.0}
-
- async def mock_get_cache_side_effect(keys, **kwargs):
- return [{"tag:tag1": mock_tag1_obj, "tag:tag2": mock_tag2_obj}.get(key) for key in keys]
-
- set_cache_called: Final = asyncio.Event()
-
- async def _record_set_cache(*args: object, **kwargs: object) -> None:
- set_cache_called.set()
-
- with patch.object(
- cache, "async_batch_get_cache", new=AsyncMock(side_effect=mock_get_cache_side_effect)
- ) as mock_get_cache:
- with patch.object(
- cache, "async_set_cache_pipeline", new=AsyncMock(side_effect=_record_set_cache)
- ) as mock_set_cache:
- await litellm.proxy.proxy_server.update_cache(
- token=None,
- user_id=None,
- end_user_id=None,
- team_id=None,
- response_cost=5.0,
- parent_otel_span=None,
- tags=["tag1", "tag2"],
- )
-
- assert await asyncio.wait_for(set_cache_called.wait(), timeout=5)
-
- mock_get_cache.assert_awaited_once_with(keys=["tag:tag1", "tag:tag2"], parent_otel_span=None, throttle_redis=False)
- mock_set_cache.assert_awaited_once()
-
- call_args = mock_set_cache.call_args
- cache_list = call_args.kwargs["cache_list"]
-
- assert len(cache_list) == 2
-
- tag_updates = {cache_key: cache_value for cache_key, cache_value in cache_list}
- assert "tag:tag1" in tag_updates
- assert "tag:tag2" in tag_updates
- assert tag_updates["tag:tag1"]["spend"] == 15.0
- assert tag_updates["tag:tag2"]["spend"] == 25.0
+ assert cache.in_memory_cache.get_cache("tag:tag1")["spend"] == 15.0
+ assert cache.in_memory_cache.get_cache("tag:tag2")["spend"] == 25.0
@pytest.mark.asyncio
@@ -6306,41 +6244,25 @@ async def test_update_cache_pipeline_honors_user_api_key_cache_ttl():
a hardcoded 60s, otherwise every priced request resets an active key's cache
entry back to 60s and the configured TTL is never observed.
"""
- from litellm.caching.caching import DualCache
+ from litellm.caching.in_memory_cache import InMemoryCache
+ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
- original_cache = litellm.proxy.proxy_server.user_api_key_cache
- cache = DualCache(default_in_memory_ttl=300)
- setattr(litellm.proxy.proxy_server, "user_api_key_cache", cache)
- set_cache_called: Final = asyncio.Event()
+ cache = UserApiKeyCache(in_memory_cache=InMemoryCache(clock=lambda: 1000.0), default_in_memory_ttl=300)
+ cache.in_memory_cache.cache_dict["tag:active-tag"] = {"tag_name": "active-tag", "spend": 1.0}
- async def _record_set_cache(*args: object, **kwargs: object) -> None:
- set_cache_called.set()
+ await litellm.proxy.proxy_server.update_cache(
+ token=None,
+ user_id=None,
+ end_user_id=None,
+ team_id=None,
+ response_cost=5.0,
+ parent_otel_span=None,
+ tags=["active-tag"],
+ cache=cache,
+ )
- try:
- with patch.object(
- cache,
- "async_batch_get_cache",
- new=AsyncMock(return_value=[{"tag_name": "active-tag", "spend": 1.0}]),
- ):
- with patch.object(
- cache, "async_set_cache_pipeline", new=AsyncMock(side_effect=_record_set_cache)
- ) as mock_set_cache:
- await litellm.proxy.proxy_server.update_cache(
- token=None,
- user_id=None,
- end_user_id=None,
- team_id=None,
- response_cost=5.0,
- parent_otel_span=None,
- tags=["active-tag"],
- )
-
- assert await asyncio.wait_for(set_cache_called.wait(), timeout=5)
-
- mock_set_cache.assert_awaited_once()
- assert mock_set_cache.call_args.kwargs["ttl"] == 300
- finally:
- setattr(litellm.proxy.proxy_server, "user_api_key_cache", original_cache)
+ assert cache.in_memory_cache.ttl_dict["tag:active-tag"] == 1300.0
+ assert cache.in_memory_cache.get_cache("tag:active-tag")["spend"] == 6.0
@pytest.mark.asyncio
@@ -6393,63 +6315,6 @@ async def test_spend_tracking_never_writes_the_auth_object_back():
setattr(litellm.proxy.proxy_server, "user_api_key_cache", original_cache)
-@pytest.mark.asyncio
-async def test_update_cache_global_proxy_spend_scalar_stays_shared():
- """
- The proxy-wide spend estimate must keep flowing to Redis when the spend
- writeback goes per-pod: the global max_budget check reads the
- ``{litellm_proxy_admin_name}:spend`` cache entry between authoritative DB
- reloads, so keeping it pod-local would let traffic spread across replicas
- exceed the proxy budget by roughly a factor of the replica count within a
- cache TTL. Sharing this scalar is safe because it carries no limits or
- permissions, so it cannot resurrect an invalidated auth blob.
- """
- from litellm.caching.caching import DualCache
-
- admin_name = litellm.proxy.proxy_server.litellm_proxy_admin_name
- global_key = "{}:spend".format(admin_name)
-
- def fake_get(key):
- if key == "user-lit":
- return {"user_id": "user-lit", "spend": 1.0}
- if key == global_key:
- return 10.0
- return None
-
- async def fake_batch_get(keys, **kwargs):
- return [fake_get(key) for key in keys]
-
- original_cache = litellm.proxy.proxy_server.user_api_key_cache
- cache = DualCache(default_in_memory_ttl=300)
- setattr(litellm.proxy.proxy_server, "user_api_key_cache", cache)
- try:
- with patch.object(cache, "async_batch_get_cache", new=AsyncMock(side_effect=fake_batch_get)):
- with patch.object(cache, "async_set_cache_pipeline", new=AsyncMock()) as mock_set_cache:
- await litellm.proxy.proxy_server.update_cache(
- token=None,
- user_id="user-lit",
- end_user_id=None,
- team_id=None,
- response_cost=5.0,
- parent_otel_span=None,
- )
-
- pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
- if pending:
- await asyncio.wait(pending, timeout=5)
-
- calls = mock_set_cache.await_args_list
- local_keys = [k for c in calls if c.kwargs.get("local_only") is True for k, _ in c.kwargs["cache_list"]]
- shared_keys = [
- k for c in calls if c.kwargs.get("local_only") is not True for k, _ in c.kwargs["cache_list"]
- ]
- assert "user-lit" in local_keys
- assert global_key not in local_keys
- assert shared_keys == [global_key]
- finally:
- setattr(litellm.proxy.proxy_server, "user_api_key_cache", original_cache)
-
-
@pytest.mark.asyncio
async def test_init_sso_settings_in_db():
"""
@@ -15720,46 +15585,3 @@ async def test_spend_capture_rate_check_job_clears_the_gauge_once_the_setting_is
call(api_provider="openai", capture_rate=None),
]
-
-@pytest.mark.asyncio
-async def test_update_cache_reads_and_writes_declare_the_auth_objects_key_family():
- """The post-call spend write-back reads and rewrites the cached auth objects, so its
- Redis spans must read ``redis.mget auth_objects`` / ``redis.set auth_objects`` (the key
- family the auth phase declares) and the global spend scalar ``redis.set spend_counters``,
- never a bare ``redis.mget`` with no owner."""
- from litellm.caching.caching import DualCache
-
- original_cache = litellm.proxy.proxy_server.user_api_key_cache
- cache = DualCache()
- setattr(litellm.proxy.proxy_server, "user_api_key_cache", cache)
- seen: list[tuple[str, str | None]] = []
-
- async def _mget(keys, **_kwargs):
- seen.append(("mget", current_service_target()))
- return [{"user_id": "u1", "spend": 1.0} for _ in keys]
-
- async def _set_pipeline(**_kwargs):
- seen.append(("set", current_service_target()))
-
- try:
- with (
- patch.object(cache, "async_batch_get_cache", new=AsyncMock(side_effect=_mget)),
- patch.object(cache, "async_set_cache_pipeline", new=AsyncMock(side_effect=_set_pipeline)),
- ):
- await litellm.proxy.proxy_server.update_cache(
- token=None,
- user_id="u1",
- end_user_id=None,
- team_id=None,
- response_cost=2.0,
- parent_otel_span=None,
- )
- pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
- if pending:
- await asyncio.wait(pending, timeout=5)
- finally:
- setattr(litellm.proxy.proxy_server, "user_api_key_cache", original_cache)
-
- assert seen, "update_cache must touch the cache for a priced user request"
- assert {target for _, target in seen} == {"auth_objects"}
- assert current_service_target() is None
diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py
index 8d3e341096c..60a75c688c4 100644
--- a/tests/unit/test_internal_context.py
+++ b/tests/unit/test_internal_context.py
@@ -87,10 +87,10 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset(
)
_CACHE_CALL: Final = re.compile(
- r"\.(?:async_)?(?:get_cache|set_cache|batch_get_cache|batch_get_cache_shared|increment_cache|increment"
+ r"\.(?:async_)?(?:get_cache|set_cache|batch_get_cache|batch_get_cache_shared|increment_cache|increment_if_present|increment"
r"|set_cache_pipeline|set_cache_pipeline_with_ttls|set_cache_sadd|delete_cache|batch_set_cache|increment_pipeline"
r"|rpush|lpop|scan_iter|get_ttl|mget)\("
- r"|\b(?:reserve_redis_batch_reads|declare_batch_get|_prepare_batch_get)\("
+ r"|\b(?:reserve_redis_batch_reads|_prepare_batch_get)\("
r"|\bbatch\.(?:set|delete|script|increment)\("
)
_DECLARES_TARGET: Final = re.compile(r"\b(?:with_service_target|service_target|response_cache_phase)\(")
From 52d5a51fcb3f6bf343dcdda1b995da916569ed4d Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
<158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 20 Oct 2026 17:10:16 +0000
Subject: [PATCH 004/203] test(cost): run azure_ai claude bridge pairs, add
anthropic usage speed and vertex gemini audio priority price (#47582)
* test(cost): run the azure_ai claude bridge cost parity pairs
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test(cost): add anthropic usage speed and vertex gemini audio priority price
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---------
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
.../integration/cost_calculation/cases/anthropic.py | 12 ++++++++----
tests/integration/cost_calculation/catalog.py | 1 +
tests/integration/cost_calculation/cost_map.py | 1 +
.../cost_calculation/cost_tracking_case.py | 1 +
4 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/tests/integration/cost_calculation/cases/anthropic.py b/tests/integration/cost_calculation/cases/anthropic.py
index 5ad4afa9530..776c5b51dd4 100644
--- a/tests/integration/cost_calculation/cases/anthropic.py
+++ b/tests/integration/cost_calculation/cases/anthropic.py
@@ -4478,6 +4478,7 @@ CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
+ "speed": "standard",
},
"diagnostics": None,
},
@@ -4502,7 +4503,7 @@ CLAUDE_SONNET_5_5_MESSAGES_STREAM: Final = CostTrackingTestCase(
response=SseResponse(
content_type="text/event-stream",
frames=(
- 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-sonnet-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}',
+ 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-sonnet-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"diagnostics":null}}',
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}',
'event: ping\ndata: {"type":"ping"}',
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}}',
@@ -4877,6 +4878,7 @@ CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
+ "speed": "standard",
},
"diagnostics": None,
},
@@ -4901,7 +4903,7 @@ CLAUDE_HAIKU_5_5_MESSAGES_STREAM: Final = CostTrackingTestCase(
response=SseResponse(
content_type="text/event-stream",
frames=(
- 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-haiku-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}',
+ 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-haiku-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"diagnostics":null}}',
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}',
'event: ping\ndata: {"type":"ping"}',
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}}',
@@ -5639,6 +5641,7 @@ CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
+ "speed": "standard",
},
"diagnostics": None,
},
@@ -5663,7 +5666,7 @@ CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = CostTrackingTestCase(
response=SseResponse(
content_type="text/event-stream",
frames=(
- 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-fable-5-1","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}',
+ 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-fable-5-1","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"diagnostics":null}}',
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
'event: ping\ndata: {"type":"ping"}',
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted "}}',
@@ -5974,6 +5977,7 @@ CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
+ "speed": "standard",
},
"diagnostics": None,
},
@@ -5998,7 +6002,7 @@ CLAUDE_OPUS_5_5_MESSAGES_STREAM: Final = CostTrackingTestCase(
response=SseResponse(
content_type="text/event-stream",
frames=(
- 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-opus-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}',
+ 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-opus-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"diagnostics":null}}',
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}',
'event: ping\ndata: {"type":"ping"}',
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}}',
diff --git a/tests/integration/cost_calculation/catalog.py b/tests/integration/cost_calculation/catalog.py
index 568a89ad967..2c1d9d9e22c 100644
--- a/tests/integration/cost_calculation/catalog.py
+++ b/tests/integration/cost_calculation/catalog.py
@@ -65,6 +65,7 @@ BRIDGE_COST_PARITY_CASES: Final[tuple[BridgeCostParityTestCase, ...]] = (
*anthropic.BRIDGE_COST_PARITY,
*gemini.BRIDGE_COST_PARITY,
*vertex_ai.BRIDGE_COST_PARITY,
+ *misc.BRIDGE_COST_PARITY,
)
_PARITY_LEGS: Final = frozenset(leg.name for pair in PARITY_CASES for leg in (pair.plain, pair.streamed)) | frozenset(
leg.name for pair in BRIDGE_COST_PARITY_CASES for leg in (pair.native, pair.bridged)
diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py
index ba7be7647da..012d847e676 100644
--- a/tests/integration/cost_calculation/cost_map.py
+++ b/tests/integration/cost_calculation/cost_map.py
@@ -597,6 +597,7 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
"cache_read_input_token_cost_priority": 2.16e-07,
"google_maps_grounding_cost_per_query": 0.0168,
"input_cost_per_audio_token": 2.4e-06,
+ "input_cost_per_audio_token_priority": 4.32e-06,
"input_cost_per_token": 2.4e-06,
"input_cost_per_token_flex": 1.2e-06,
"input_cost_per_token_priority": 4.32e-06,
diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py
index 3efa97407f7..e98f6a44222 100644
--- a/tests/integration/cost_calculation/cost_tracking_case.py
+++ b/tests/integration/cost_calculation/cost_tracking_case.py
@@ -94,6 +94,7 @@ class CostMapEntry(BaseModel):
tiered_pricing: tuple[TieredPrice, ...] | None = None
output_cost_per_reasoning_token: float | None = None
input_cost_per_audio_token: float | None = None
+ input_cost_per_audio_token_priority: float | None = None
input_cost_per_second: float | None = None
output_cost_per_second: float | None = None
input_cost_per_character: float | None = None
From 3c0e08146d80513ccddf19a1a9e52a2a394480ea Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 10:19:22 -0700
Subject: [PATCH 005/203] test(perf): set every perf ceiling to 1.2x the lane
median (#47590)
* test(perf): set every perf ceiling to 1.2x the lane median
* test(perf): derive the upgrade verdict test's samples from the case ceilings
* test(perf): keep memory_growth_megabytes at 100 and 600, the metric swings 60 MB between reruns
* test(perf): keep small litellm_overhead_seconds p99 at 1.2x the lane max, it is the 3rd slowest of 300 requests
---------
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
.../perf/cases/mixed_size_requests_2rps.py | 34 +++++++++----------
tests/e2e/perf/cases/small_bodies_5rps.py | 28 +++++++--------
tests/e2e_harness/perf/test_upgrade.py | 25 ++++++++++----
3 files changed, 49 insertions(+), 38 deletions(-)
diff --git a/tests/e2e/perf/cases/mixed_size_requests_2rps.py b/tests/e2e/perf/cases/mixed_size_requests_2rps.py
index d4f1fdddc9c..3acd8671acd 100644
--- a/tests/e2e/perf/cases/mixed_size_requests_2rps.py
+++ b/tests/e2e/perf/cases/mixed_size_requests_2rps.py
@@ -62,24 +62,24 @@ CASE: Final = PerformanceTestCase(
),
limits=(
Limit("unexpected_status_count", max=3),
- Limit("gateway_added_seconds", percentile=50, max=0.58),
- Limit("gateway_added_seconds", percentile=99, max=6.1),
- Limit("litellm_overhead_seconds", percentile=50, max=0.1),
- Limit("litellm_overhead_seconds", percentile=99, max=3.4),
- Limit("litellm_response_seconds", percentile=50, max=1.0),
- Limit("litellm_response_seconds", percentile=99, max=3.8),
- Limit("prometheus_redis_latency_mean_seconds", max=0.005),
- Limit("postgres_active_sessions", max=0.016),
- Limit("postgres_slowest_statement_seconds", max=0.25),
- Limit("postgres_peak_connections", max=45),
- Limit("time_to_first_token_seconds", percentile=50, max=2.7),
- Limit("time_to_first_token_seconds", percentile=99, max=8.8),
+ Limit("gateway_added_seconds", percentile=50, max=0.29),
+ Limit("gateway_added_seconds", percentile=99, max=2.7),
+ Limit("litellm_overhead_seconds", percentile=50, max=0.052),
+ Limit("litellm_overhead_seconds", percentile=99, max=1.8),
+ Limit("litellm_response_seconds", percentile=50, max=0.93),
+ Limit("litellm_response_seconds", percentile=99, max=2.7),
+ Limit("prometheus_redis_latency_mean_seconds", max=0.002),
+ Limit("postgres_active_sessions", max=0.0094),
+ Limit("postgres_slowest_statement_seconds", max=0.042),
+ Limit("postgres_peak_connections", max=23),
+ Limit("time_to_first_token_seconds", percentile=50, max=1.56),
+ Limit("time_to_first_token_seconds", percentile=99, max=4.5),
Limit("memory_growth_megabytes", max=600),
- Limit("cpu_seconds_per_request", max=1.26),
- Limit("postgres_calls_per_request", max=3.5),
- Limit("postgres_max_statement_per_request", max=0.5),
- Limit("postgres_max_background_statement_per_request", max=0.9),
- Limit("redis_calls_per_request", max=21.0),
+ Limit("cpu_seconds_per_request", max=0.57),
+ Limit("postgres_calls_per_request", max=2.64),
+ Limit("postgres_max_statement_per_request", max=0.43),
+ Limit("postgres_max_background_statement_per_request", max=0.86),
+ Limit("redis_calls_per_request", max=15.4),
),
background_cap_exempt_tables=frozenset({"LiteLLM_DailyGuardrailMetrics"}),
)
diff --git a/tests/e2e/perf/cases/small_bodies_5rps.py b/tests/e2e/perf/cases/small_bodies_5rps.py
index f93c64afba4..4bf58cde3a5 100644
--- a/tests/e2e/perf/cases/small_bodies_5rps.py
+++ b/tests/e2e/perf/cases/small_bodies_5rps.py
@@ -64,21 +64,21 @@ CASE: Final = PerformanceTestCase(
),
limits=(
Limit("unexpected_status_count", max=3),
- Limit("gateway_added_seconds", percentile=50, max=0.29),
- Limit("litellm_overhead_seconds", percentile=50, max=0.012),
+ Limit("gateway_added_seconds", percentile=50, max=0.165),
+ Limit("litellm_overhead_seconds", percentile=50, max=0.006),
Limit("litellm_overhead_seconds", percentile=99, max=0.35),
- Limit("litellm_response_seconds", percentile=50, max=1.0),
- Limit("litellm_response_seconds", percentile=99, max=2.0),
- Limit("prometheus_redis_latency_mean_seconds", max=0.006),
- Limit("postgres_active_sessions", max=0.01),
- Limit("postgres_slowest_statement_seconds", max=0.025),
- Limit("postgres_peak_connections", max=45),
- Limit("time_to_first_token_seconds", percentile=99, max=3.7),
+ Limit("litellm_response_seconds", percentile=50, max=0.91),
+ Limit("litellm_response_seconds", percentile=99, max=1.6),
+ Limit("prometheus_redis_latency_mean_seconds", max=0.0026),
+ Limit("postgres_active_sessions", max=0.0052),
+ Limit("postgres_slowest_statement_seconds", max=0.011),
+ Limit("postgres_peak_connections", max=23),
+ Limit("time_to_first_token_seconds", percentile=99, max=2.05),
Limit("memory_growth_megabytes", max=100),
- Limit("postgres_calls_per_request", max=1.8),
- Limit("postgres_max_statement_per_request", max=0.5),
- Limit("postgres_max_background_statement_per_request", max=0.9),
- Limit("redis_calls_per_request", max=18),
- Limit("cpu_seconds_per_request", max=0.29),
+ Limit("postgres_calls_per_request", max=1.31),
+ Limit("postgres_max_statement_per_request", max=0.27),
+ Limit("postgres_max_background_statement_per_request", max=0.92),
+ Limit("redis_calls_per_request", max=13.8),
+ Limit("cpu_seconds_per_request", max=0.15),
),
)
diff --git a/tests/e2e_harness/perf/test_upgrade.py b/tests/e2e_harness/perf/test_upgrade.py
index 89e3a620f48..1f2bc8c46cb 100644
--- a/tests/e2e_harness/perf/test_upgrade.py
+++ b/tests/e2e_harness/perf/test_upgrade.py
@@ -295,19 +295,30 @@ def test_invalid_200_is_not_an_allowed_non_200(failed: WireRecord) -> None:
_ = Probe().monitor().poll((failed,))
+def _ceiling(metric: str, percentile: int | None = None) -> float:
+ return next(
+ limit.max for limit in upgrade_case().limits if limit.metric == metric and limit.percentile == percentile
+ )
+
+
@pytest.mark.parametrize(
- "failures,overhead,ttft,growth,expected",
+ "over,expected",
[
- (3, 0.29, 3.7, 100, ()),
- (4, 0.29, 3.7, 100, ("unexpected_status_count",)),
- (3, 0.30, 3.7, 100, ("gateway_added_seconds percentile 50",)),
- (3, 0.29, 3.8, 100, ("time_to_first_token_seconds percentile 99",)),
- (3, 0.29, 3.7, 101, ("memory_growth_megabytes",)),
+ ("", ()),
+ ("unexpected_status_count", ("unexpected_status_count",)),
+ ("gateway_added_seconds", ("gateway_added_seconds percentile 50",)),
+ ("time_to_first_token_seconds", ("time_to_first_token_seconds percentile 99",)),
+ ("memory_growth_megabytes", ("memory_growth_megabytes",)),
],
)
def test_upgrade_verdict_uses_regular_limits_without_cpu_or_datastore_samples(
- failures: int, overhead: float, ttft: float, growth: float, expected: tuple[str, ...]
+ over: str, expected: tuple[str, ...]
) -> None:
+ """Each sample sits exactly on the case's ceiling, so only the one nudged past it goes over."""
+ failures: Final = int(_ceiling("unexpected_status_count")) + (over == "unexpected_status_count")
+ overhead: Final = _ceiling("gateway_added_seconds", 50) * (1.01 if over == "gateway_added_seconds" else 1)
+ ttft: Final = _ceiling("time_to_first_token_seconds", 99) * (1.01 if over == "time_to_first_token_seconds" else 1)
+ growth: Final = _ceiling("memory_growth_megabytes") + (over == "memory_growth_megabytes")
nonstream: Final = tuple(record(i).model_copy(update={"total_s": overhead, "mock_delay_s": 0}) for i in range(100))
stream: Final = record(100).model_copy(update={"stream": True, "first_token_s": ttft})
observed: Final = Observation(
From 2484f5dd6d5e274778e1333d29548c0b14cafd7e Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 17:39:06 +0000
Subject: [PATCH 006/203] test(integration): call the renamed metadata
deployment helper in the Mantle count_tokens wire tests (#47592)
---
.../providers/test_bedrock_mantle_count_tokens_wire.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py
index 25f46be4c5d..a4542f1f604 100644
--- a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py
+++ b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py
@@ -1683,8 +1683,8 @@ def test_responses_input_tokens_sends_mantle_the_tool_use_invoke_inference_sends
wire_server(_mantle(), port=mantle_port) as mantle,
gateway.scenario() as scenario,
):
- counted_model: Final = _fallback_deployment(scenario, runtime.url)
- invoked_model: Final = _fallback_deployment(scenario, runtime.url, f"invoke/{_OPUS}")
+ counted_model: Final = _metadata_deployment(scenario, runtime.url)
+ invoked_model: Final = _metadata_deployment(scenario, runtime.url, f"invoke/{_OPUS}")
generated: Final = gateway.request(
"POST", "/v1/responses", {**_FORCED_TOOL_CHOICE, "model": invoked_model, "max_output_tokens": 16}
)
@@ -1729,7 +1729,7 @@ def test_nova_count_sends_the_tool_specs_converse_inference_sends(counting_proxy
path, body, count_field = _NOVA_COUNTS[route]
gateway: Final = counting_proxy.gateway
with wire_server(_counting_every_model) as runtime, gateway.scenario() as scenario:
- model: Final = _fallback_deployment(scenario, runtime.url, _NOVA)
+ model: Final = _metadata_deployment(scenario, runtime.url, _NOVA)
generated: Final = gateway.request(
"POST",
"/v1/chat/completions",
From ad8ca533ca2cd5fa4360d519183c1a2aaccc084a Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 10:39:29 -0700
Subject: [PATCH 007/203] feat(proxy): write one spend row per billed timed-out
retry attempt (#47416)
_failure_spend_rows yields the final row plus one re-keyed row per lifted billed_failed_attempts entry (-attempt-, the attempt's usage, cost, retry count and deployment id), each written through DBSpendUpdateWriter.update_database so it also deducts from key, team and user spend
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
.../proxy/hooks/proxy_track_cost_callback.py | 115 +++++++++++++++---
.../hooks/test_proxy_track_cost_callback.py | 59 +++++++++
2 files changed, 158 insertions(+), 16 deletions(-)
diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py
index 38a76028ddc..1bfb04ab883 100644
--- a/litellm/proxy/hooks/proxy_track_cost_callback.py
+++ b/litellm/proxy/hooks/proxy_track_cost_callback.py
@@ -55,7 +55,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 #
sanitize_error_information_for_spend_logs,
should_store_prompts_and_responses_in_spend_logs,
)
-from litellm.proxy.utils import ProxyUpdateSpend
+from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend
from litellm.types.utils import (
CallTypes,
LiteLLMBatch,
@@ -153,6 +153,73 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter:
return proxy_logging_obj.db_spend_update_writer
+def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]:
+ """The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any."""
+ attempts: Final = request_data.get("billed_failed_attempts")
+ if not isinstance(attempts, tuple):
+ return ()
+ lifted: Final = cast("tuple[object, ...]", attempts) # cast-ok: lifted by _failure_fields_to_lift, checked below
+ return tuple(attempt for attempt in lifted if isinstance(attempt, BilledFailedAttempt))
+
+
+def _failure_spend_rows(
+ request_data: Mapping[str, object], final_cost: float, late_guardrails_snapshot: object
+) -> tuple[tuple[float, Mapping[str, object]], ...]:
+ """One ``(response_cost, kwargs)`` per spend row a failure writes: the final attempt under the
+ request id, then every earlier attempt the provider also billed under its own row."""
+ final_row: Final[Mapping[str, object]] = (
+ request_data
+ if late_guardrails_snapshot is None
+ else {**request_data, "standard_logging_object": late_guardrails_snapshot}
+ )
+ return (
+ (final_cost, final_row),
+ *(
+ (attempt.response_cost, _attempt_spend_row_kwargs(request_data, attempt, position))
+ for position, attempt in enumerate(_billed_failed_attempts(request_data))
+ ),
+ )
+
+
+def _attempt_spend_row_kwargs(
+ request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int
+) -> Mapping[str, object]:
+ """The final attempt's row kwargs re-keyed for one earlier billed attempt: its own
+ ``request_id`` (``-attempt-``, since ``request_id`` is the spend-log primary key
+ and the retry count restarts after a fallback hop), its estimated usage and cost, its retry
+ count under ``attempted_retries`` and the deployment that billed it under ``model_info.id``."""
+ suffix: Final = f"-attempt-{position}"
+ raw_params: Final = cast("Mapping[str, object]", request_data.get("litellm_params") or {}) # cast-ok: request dict
+ litellm_params: Final = dict(raw_params)
+ metadata: Final = {
+ **cast("Mapping[str, object]", litellm_params.get("metadata") or {}), # cast-ok: request dict
+ "attempted_retries": attempt.attempted_retries,
+ }
+ metadata.pop("previous_models", None)
+ if attempt.deployment_id is not None:
+ metadata["model_info"] = {
+ **cast("Mapping[str, object]", metadata.get("model_info") or {}), # cast-ok: router dict
+ "id": attempt.deployment_id,
+ }
+ litellm_params["metadata"] = metadata
+ row: Final = {
+ **request_data,
+ "litellm_params": litellm_params,
+ "combined_usage_object": attempt.usage,
+ "response_cost": attempt.response_cost,
+ }
+ row.pop("billed_failed_attempts", None)
+ call_id: Final = request_data.get("litellm_call_id") or litellm_params.get("litellm_call_id")
+ if isinstance(call_id, str):
+ row["litellm_call_id"] = f"{call_id}{suffix}"
+ standard_logging_object: Final = request_data.get("standard_logging_object")
+ if isinstance(standard_logging_object, dict):
+ payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict
+ if isinstance(payload.get("id"), str):
+ row["standard_logging_object"] = {**payload, "id": f"{payload['id']}{suffix}"}
+ return row
+
+
class _ProxyDBLogger(CustomLogger):
def __init__(
self,
@@ -330,24 +397,40 @@ class _ProxyDBLogger(CustomLogger):
existing_metadata.get("standard_logging_guardrail_information"),
)
- await self._spend_writer().update_database(
- token=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict),
- response_cost=recovered_response_cost,
- user_id=user_api_key_dict.user_id,
- end_user_id=None if litellm.disable_end_user_cost_tracking else user_api_key_dict.end_user_id,
- team_id=user_api_key_dict.team_id,
- kwargs=(
- request_data
- if failure_snapshot_with_late_guardrails is None
- else {**request_data, "standard_logging_object": failure_snapshot_with_late_guardrails}
- ),
- completion_response=original_exception,
+ await self._write_failure_spend_rows(
+ user_api_key_dict=user_api_key_dict,
+ original_exception=original_exception,
start_time=actual_start_time,
- end_time=datetime.now(),
- org_id=user_api_key_dict.org_id,
- project_id=user_api_key_dict.project_id,
+ spend_rows=_failure_spend_rows(
+ cast("Mapping[str, object]", request_data), # cast-ok: request dict, copied per row below
+ recovered_response_cost,
+ failure_snapshot_with_late_guardrails,
+ ),
)
+ async def _write_failure_spend_rows(
+ self,
+ user_api_key_dict: UserAPIKeyAuth,
+ original_exception: Exception,
+ start_time: datetime,
+ spend_rows: Sequence[tuple[float, Mapping[str, object]]],
+ ) -> None:
+ """One spend row (and one key/team/user spend deduction) per ``(response_cost, kwargs)`` pair."""
+ for row_cost, row_kwargs in spend_rows:
+ await self._spend_writer().update_database(
+ token=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict),
+ response_cost=row_cost,
+ user_id=user_api_key_dict.user_id,
+ end_user_id=None if litellm.disable_end_user_cost_tracking else user_api_key_dict.end_user_id,
+ team_id=user_api_key_dict.team_id,
+ kwargs=dict(row_kwargs),
+ completion_response=original_exception,
+ start_time=start_time,
+ end_time=datetime.now(),
+ org_id=user_api_key_dict.org_id,
+ project_id=user_api_key_dict.project_id,
+ )
+
async def _PROXY_track_cost_callback(
self,
kwargs, # kwargs to completion
diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
index 1f0dd67d86b..1132d339f15 100644
--- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
+++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
@@ -95,6 +95,65 @@ async def test_async_post_call_failure_hook():
assert metadata["original_key"] == "original_value"
+@pytest.mark.asyncio
+async def test_each_earlier_billed_attempt_gets_its_own_spend_row():
+ """A timed-out request that the router retried writes one row per billed attempt: the final
+ attempt under the request id, every earlier Timeout attempt under ``-attempt-`` with its
+ own cost, attempt number and deployment."""
+ from litellm.proxy.utils import BilledFailedAttempt
+
+ logger = ProxyDBLogger()
+ user_api_key_dict = UserAPIKeyAuth(api_key="test_api_key", user_id="u1", team_id="t1")
+ usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
+ request_data = {
+ "model": "gpt-4.1",
+ "litellm_call_id": "call-1",
+ "standard_logging_object": {"id": "call-1", "model_id": "dep-final"},
+ "metadata": {
+ "attempted_retries": 2,
+ "model_info": {"id": "dep-final", "input_cost_per_token": 0.5},
+ "previous_models": ({"exception_type": "Timeout"}, {"exception_type": "Timeout"}),
+ },
+ "combined_usage_object": usage,
+ "response_cost": 0.5,
+ "billed_failed_attempts": (
+ BilledFailedAttempt(attempted_retries=0, deployment_id="dep-a", usage=usage, response_cost=0.25),
+ BilledFailedAttempt(attempted_retries=0, deployment_id="dep-final", usage=usage, response_cost=0.5),
+ ),
+ }
+
+ with patch(
+ "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database",
+ new_callable=AsyncMock,
+ ) as mock_update_database:
+ await logger.async_post_call_failure_hook(
+ request_data=request_data,
+ original_exception=litellm.Timeout("Request timed out.", model="gpt-4.1", llm_provider="openai"),
+ user_api_key_dict=user_api_key_dict,
+ )
+
+ rows = [call.kwargs for call in mock_update_database.call_args_list]
+ assert [row["response_cost"] for row in rows] == [0.5, 0.25, 0.5]
+ final_row, attempt_0, attempt_1 = (row["kwargs"] for row in rows)
+ assert final_row["litellm_call_id"] == "call-1"
+ assert final_row["litellm_params"]["metadata"]["attempted_retries"] == 2
+ assert attempt_0["litellm_call_id"] == "call-1-attempt-0"
+ assert attempt_0["standard_logging_object"]["id"] == "call-1-attempt-0"
+ assert attempt_0["litellm_params"]["metadata"]["attempted_retries"] == 0
+ assert attempt_0["litellm_params"]["metadata"]["model_info"]["id"] == "dep-a"
+ assert "previous_models" not in attempt_0["litellm_params"]["metadata"]
+ assert "billed_failed_attempts" not in attempt_0
+ assert attempt_0["combined_usage_object"] is usage
+ assert attempt_1["litellm_call_id"] == "call-1-attempt-1"
+ assert attempt_1["litellm_params"]["metadata"]["attempted_retries"] == 0
+ assert attempt_1["litellm_params"]["metadata"]["model_info"]["id"] == "dep-final"
+ assert final_row["litellm_params"]["metadata"]["model_info"]["id"] == "dep-final"
+ for row in rows:
+ assert row["token"] == "test_api_key"
+ assert row["user_id"] == "u1"
+ assert row["team_id"] == "t1"
+
+
@pytest.mark.asyncio
async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_metadata():
"""
From 208cac49f05167275d90c0430ec5b0295ceccf25 Mon Sep 17 00:00:00 2001
From: Max Azatian <66359507+HardMax71@users.noreply.github.com>
Date: Tue, 20 Oct 2026 19:39:51 +0200
Subject: [PATCH 008/203] fix(vertex_ai)!: send integer enums to Gemini instead
of dropping them (#44908)
* fix(vertex_ai): send integer enums to Gemini instead of dropping them
Gemini rejects enum values that are not strings, so _fix_enum_types removed
every non-string enum. Integer enums are now sent as type INTEGER with
format "enum" and string values, which Gemini accepts and enforces.
* fix(vertex_ai): keep converted integer enums when a tool is reused
The schema is edited in place, so a tool definition passed again already
carries the converted enum. Keep an integer enum that is already in the
format "enum" form instead of dropping it, and type the new locals.
* fix(vertex_ai): use Sequence at runtime in the enum cast
* fix(vertex_ai): treat integer-string enum values as an integer enum
A reused schema already holds its integer enum as strings. Instead of
reading `format` to recognise that, accept integer strings as well as
integers on an integer-typed enum and send both in Gemini's form. This
drops the untyped `schema.get("format")` read.
* fix(vertex_ai): map Gemini tools from a copy so a fallback keeps the caller's schema
Gemini tool mapping rewrote the caller's tools in place, so a router fallback to another provider received the Gemini-converted integer enum (string values with format enum) and returned the integer as a string
* fix(vertex_ai): send integer enums only to Gemini 3 and newer, and only ones Gemini accepts
gemini-2.5-flash stalls for minutes on a larger integer enum, so Gemini 2.x
keeps main's behavior and drops it. An enum outside int64 or longer than 1000
values is dropped too, since Gemini answers 400 for those. A nullable integer
enum written as a type list now keeps its values like the anyOf form does
* fix(vertex_ai): drop integer enum strings longer than int64 instead of failing the request
* fix(vertex_ai): drop integer enums when a request's enums pass Gemini's size limit, and send int enum members as numbers
Gemini limits the total size of all enums in one request, so the per-enum 1000-value cap still let several large integer enums through to a 400. Integer enums are now sent only while every enum value in the request adds up to 2000 characters or fewer, and an (int, Enum) member goes out as its number instead of its name
* fix(vertex_ai): send only integer enums Gemini returns exactly, and count them in Gemini token counts
* fix(vertex_ai): send Gemini integer enums without format enum, which made gemini-3.8-flash think much longer
Also skip the response cache in the repeated-request integration test, since copying tools makes the second identical request a cache hit
* fix(vertex_ai): keep dropping integer enums in Vertex batch rows, which Vertex batch cannot parse
---------
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
---
litellm/llms/gemini/common_utils.py | 2 +-
.../gemini/google_genai/transformation.py | 10 +-
litellm/llms/vertex_ai/common_utils.py | 131 +++++++++--
.../llms/vertex_ai/count_tokens/context.py | 16 +-
.../llms/vertex_ai/files/transformation.py | 1 +
.../vertex_and_google_ai_studio_gemini.py | 46 ++--
.../test_gemini_anyof_null_branches_wire.py | 5 +-
.../test_google_genai_transformation.py | 41 ++++
.../vertex_ai/count_tokens/test_context.py | 49 ++++-
.../test_vertex_ai_files_transformation.py | 40 ++++
...test_vertex_and_google_ai_studio_gemini.py | 203 +++++++++++++++++-
.../vertex_ai/test_vertex_ai_common_utils.py | 156 ++++++++++++--
12 files changed, 633 insertions(+), 67 deletions(-)
diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py
index b1a726be876..8ee7a463828 100644
--- a/litellm/llms/gemini/common_utils.py
+++ b/litellm/llms/gemini/common_utils.py
@@ -524,7 +524,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
)
if isinstance(request_view, VertexGeminiCountContextError):
return count_context_error_response(request_view, request_model, model_to_use)
- count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
+ count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools, model=model_to_use)
if isinstance(count_context, VertexGeminiCountContextError):
return count_context_error_response(count_context, request_model, model_to_use)
blocks_error: Final = unconvertible_messages_error(
diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py
index 196a19a141f..bc3180a2129 100644
--- a/litellm/llms/gemini/google_genai/transformation.py
+++ b/litellm/llms/gemini/google_genai/transformation.py
@@ -15,8 +15,9 @@ from litellm.llms.base_llm.google_genai.transformation import (
from litellm.llms.vertex_ai.common_utils import (
build_vertex_schema,
supports_response_json_schema,
+ with_integer_enums_that_fit,
)
-from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
+from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig, VertexLLM
from litellm.types.router import GenericLiteLLMParams
if TYPE_CHECKING:
@@ -327,8 +328,11 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
else:
if json_schema_key is not None:
generate_content_config_dict.pop(json_schema_key)
- generate_content_config_dict[schema_key] = build_vertex_schema(
- parameters=deepcopy(value), add_property_ordering=True
+ generate_content_config_dict[schema_key] = with_integer_enums_that_fit(
+ lambda send_integer_enums: build_vertex_schema(
+ parameters=deepcopy(value), add_property_ordering=True, send_integer_enums=send_integer_enums
+ ),
+ VertexGeminiConfig.is_gemini_3_or_newer(model),
)
def transform_generate_content_request(
diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py
index f85a5a8cda9..27f2aedb300 100644
--- a/litellm/llms/vertex_ai/common_utils.py
+++ b/litellm/llms/vertex_ai/common_utils.py
@@ -1,11 +1,12 @@
import re
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Iterable, Mapping, Sequence
from copy import deepcopy
from datetime import datetime, timezone
from enum import Enum
from functools import lru_cache
+from itertools import accumulate, chain, repeat, takewhile
from types import MappingProxyType
-from typing import Any, Final, Literal, cast, get_type_hints
+from typing import Any, Final, Literal, TypeVar, cast, get_type_hints
import httpx
from pydantic import JsonValue, TypeAdapter, ValidationError
@@ -666,12 +667,89 @@ def _fix_enum_empty_strings(schema, depth=0):
_fix_enum_empty_strings(items, depth=depth + 1)
-def _fix_enum_types(schema, depth=0):
- """Remove `enum` fields when the schema type is not string.
+GEMINI_MAX_ENUM_CHARACTERS: Final = 2000
+_GEMINI_EXACT_INT_MAX: Final = 2**53 - 1
- Gemini / Vertex APIs only allow enums for string-typed fields. When an enum
- is present on a non-string typed property (or when `anyOf` types do not
- include a string type), remove the enum to avoid provider validation errors.
+
+def _gemini_integer_enum_value(value: object) -> str | None:
+ match value:
+ case bool():
+ return None
+ case int() if -_GEMINI_EXACT_INT_MAX <= value <= _GEMINI_EXACT_INT_MAX:
+ return str(int(value))
+ case str() if (
+ re.fullmatch(r"-?[0-9]{1,16}", value) and -_GEMINI_EXACT_INT_MAX <= int(value) <= _GEMINI_EXACT_INT_MAX
+ ):
+ return str(int(value))
+ case _:
+ return None
+
+
+def _gemini_integer_enum(values: Sequence[object]) -> tuple[str, ...] | None:
+ if not values:
+ return None
+ converted: Final = tuple(text for value in values if (text := _gemini_integer_enum_value(value)) is not None)
+ return converted if len(converted) == len(values) else None
+
+
+def _json_children(node: JsonValue) -> Iterable[JsonValue]:
+ match node:
+ case dict():
+ return node.values()
+ case list():
+ return node
+ case _:
+ return ()
+
+
+def _next_json_level(level: tuple[JsonValue, ...], _: None) -> tuple[JsonValue, ...]:
+ return tuple(chain.from_iterable(_json_children(node) for node in level))
+
+
+def _enum_size(node: JsonValue) -> tuple[bool, int] | None:
+ match node:
+ case {"enum": list() as values}:
+ return (str(node.get("type")).lower() == "integer", sum(len(str(value)) for value in values))
+ case _:
+ return None
+
+
+def _integer_enums_fit(schema: JsonValue) -> bool:
+ levels: Final = takewhile(bool, accumulate(repeat(None), _next_json_level, initial=(schema,)))
+ enum_sizes: Final = tuple(size for node in chain.from_iterable(levels) if (size := _enum_size(node)) is not None)
+ return (
+ not any(is_integer for is_integer, _ in enum_sizes)
+ or sum(characters for _, characters in enum_sizes) <= GEMINI_MAX_ENUM_CHARACTERS
+ )
+
+
+_GeminiRequestPart: Final = TypeVar("_GeminiRequestPart")
+
+
+def with_integer_enums_that_fit(
+ build: Callable[[bool], _GeminiRequestPart], send_integer_enums: bool
+) -> _GeminiRequestPart:
+ if not send_integer_enums:
+ return build(False)
+ with_integer_enums: Final = build(True)
+ if _integer_enums_fit(
+ cast(JsonValue, with_integer_enums) # cast-ok: built tools and schemas are JSON request data
+ ):
+ return with_integer_enums
+ verbose_logger.debug(
+ "Gemini integer enums dropped: the request's enum values exceed %s characters", GEMINI_MAX_ENUM_CHARACTERS
+ )
+ return build(False)
+
+
+def _fix_enum_types(schema, depth=0, send_integer_enums: bool = False):
+ """Make `enum` fields acceptable to Gemini / Vertex.
+
+ Gemini / Vertex APIs only accept string enum values. With `send_integer_enums`,
+ an integer enum of values a double holds exactly is sent as type INTEGER with the
+ values as strings; the model still returns an integer.
+ On any other non-string typed property (or when `anyOf` types do not include a
+ string type), the enum is removed to avoid provider validation errors.
"""
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError(f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema.")
@@ -696,26 +774,37 @@ def _fix_enum_types(schema, depth=0):
break
if not keep_enum:
- schema.pop("enum", None)
+ is_integer_type: Final = isinstance(schema_type, str) and schema_type.lower() == "integer"
+ integer_enum: Final = (
+ _gemini_integer_enum(cast(Sequence[object], schema["enum"])) # cast-ok: an untyped JSON list
+ if send_integer_enums and is_integer_type
+ else None
+ )
+ if integer_enum is not None:
+ schema["enum"] = list(integer_enum) # rebind-ok: this function edits the schema in place by design
+ else:
+ schema.pop("enum", None)
# Recurse into nested structures
properties: Final = schema.get("properties", None)
if properties is not None:
for _, value in properties.items():
- _fix_enum_types(value, depth=depth + 1)
+ _fix_enum_types(value, depth=depth + 1, send_integer_enums=send_integer_enums)
items: Final = schema.get("items", None)
if items is not None:
- _fix_enum_types(items, depth=depth + 1)
+ _fix_enum_types(items, depth=depth + 1, send_integer_enums=send_integer_enums)
anyof = schema.get("anyOf", None)
if anyof is not None and isinstance(anyof, list):
for item in anyof:
if isinstance(item, dict):
- _fix_enum_types(item, depth=depth + 1)
+ _fix_enum_types(item, depth=depth + 1, send_integer_enums=send_integer_enums)
-def build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
+def build_vertex_schema(
+ parameters: dict, add_property_ordering: bool = False, send_integer_enums: bool = False
+) -> dict[str, object]:
"""
This is a modified version of https://github.com/google-gemini/generative-ai-python/blob/8f77cc6ac99937cd3a81299ecf79608b91b06bbb/google/generativeai/types/content_types.py#L419
@@ -725,6 +814,7 @@ def build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
parameters: dict - the json schema to build from
add_property_ordering: bool - whether to add propertyOrdering to the schema. This is only applicable to schemas for structured outputs. See
set_schema_property_ordering for more details.
+ send_integer_enums: bool - whether to send integer enums with string values instead of removing them
Returns:
parameters: dict - the input parameters, modified in place
"""
@@ -750,7 +840,7 @@ def build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
_fix_enum_empty_strings(parameters)
# Remove enums for non-string typed fields (Gemini requires enum only on strings)
- _fix_enum_types(parameters)
+ _fix_enum_types(parameters, send_integer_enums=send_integer_enums)
# Handle empty items objects
process_items(parameters)
@@ -1068,6 +1158,17 @@ def _convert_schema_types(schema, depth=0):
if field in schema:
item_schema[field] = deepcopy(schema[field])
any_of.append(item_schema)
+ elif t == "integer" and "enum" in schema and isinstance(schema["enum"], list):
+ any_of.append(
+ {
+ "type": t,
+ "enum": [
+ value
+ for value in cast(list[object], schema["enum"]) # cast-ok: an untyped JSON list
+ if value is not None
+ ],
+ }
+ )
else:
# For primitive types, only include the type
any_of.append({"type": t})
@@ -1454,7 +1555,9 @@ class VertexAITokenCounter(BaseTokenCounter):
)
if isinstance(request_view, VertexGeminiCountContextError):
return count_context_error_response(request_view, request_model, model_to_use)
- count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
+ count_context: Final = vertex_gemini_count_context(
+ system=request_view.system, tools=tools, model=model_to_use
+ )
if isinstance(count_context, VertexGeminiCountContextError):
return count_context_error_response(count_context, request_model, model_to_use)
blocks_error: Final = unconvertible_messages_error(
diff --git a/litellm/llms/vertex_ai/count_tokens/context.py b/litellm/llms/vertex_ai/count_tokens/context.py
index b13f72e96fe..2579e857bd9 100644
--- a/litellm/llms/vertex_ai/count_tokens/context.py
+++ b/litellm/llms/vertex_ai/count_tokens/context.py
@@ -78,6 +78,7 @@ def _is_function_tool(tool: Mapping[str, JsonValue]) -> bool:
def _map_function_tools(
tools: object | None,
+ model: str,
) -> tuple[Mapping[str, object], ...] | None | VertexGeminiCountContextError:
try:
tool_values: Final = _TOOLS.validate_python(tools or ())
@@ -89,15 +90,21 @@ def _map_function_tools(
if not all(_has_object_schema(tool) for tool in function_tools):
return _INVALID_TOOLS
+ from litellm.llms.vertex_ai.common_utils import with_integer_enums_that_fit
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
chat_tools: Final = tuple(
chat_tool for tool in function_tools for chat_tool in chat_completion_tools_for(tool)
) # comprehension-ok: flattens each tool's translation
+ config: Final = VertexGeminiConfig()
try:
- mapped_tools: Final = VertexGeminiConfig().map_function(
- value=[dict(tool) for tool in chat_tools],
- optional_params={},
+ mapped_tools: Final = with_integer_enums_that_fit(
+ lambda send_integer_enums: config.map_function(
+ value=[dict(tool) for tool in chat_tools],
+ optional_params={},
+ send_integer_enums=send_integer_enums,
+ ),
+ VertexGeminiConfig.is_gemini_3_or_newer(model),
)
except (AttributeError, TypeError, ValueError):
return _INVALID_TOOLS
@@ -110,11 +117,12 @@ def _map_function_tools(
def vertex_gemini_count_context(
system: object | None,
tools: object | None,
+ model: str,
) -> VertexGeminiCountContext | VertexGeminiCountContextError:
system_parts: Final = _system_parts(system)
if isinstance(system_parts, VertexGeminiCountContextError):
return system_parts
- function_tools: Final = _map_function_tools(tools)
+ function_tools: Final = _map_function_tools(tools, model)
if isinstance(function_tools, VertexGeminiCountContextError):
return function_tools
return VertexGeminiCountContext(
diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py
index 396acd270fa..ea71e5fef50 100644
--- a/litellm/llms/vertex_ai/files/transformation.py
+++ b/litellm/llms/vertex_ai/files/transformation.py
@@ -1118,6 +1118,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
non_default_params=openai_request_body,
optional_params={},
drop_params=False,
+ allow_integer_enums=False,
)
return vertex_params
diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
index 114b9cd80ae..ec40cdfd11f 100644
--- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
+++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
@@ -95,6 +95,7 @@ from ..common_utils import (
build_json_schema,
build_vertex_schema,
supports_response_json_schema,
+ with_integer_enums_that_fit,
)
from ..context_caching.known_cached_contents import known_cached_contents
from ..context_caching.rejected_cache_retry import (
@@ -692,13 +693,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return googleSearch, googleSearchRetrieval, enterpriseWebSearch, urlContext
- def _map_function(self, value: list[dict], optional_params: dict) -> list[Tools]:
+ def _map_function(self, value: list[dict], optional_params: dict, send_integer_enums: bool = False) -> list[Tools]:
"""
Map OpenAI-style tools/functions to Vertex AI format.
Args:
value: List of tool definitions
optional_params: Request-scoped parameters to store retrieval config
+ send_integer_enums: Whether to send integer enums with string values
Returns:
List of mapped tools in Vertex AI format
@@ -711,18 +713,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
googleSearch: dict | None = None
googleSearchRetrieval: dict | None = None
enterpriseWebSearch: dict | None = None
- urlContext: dict | None = None
+ urlContext: dict[str, object] | None = None
code_execution: dict | None = None
googleMaps: dict | None = None
google_maps_retrieval_config: dict | None = None
computerUse: dict | None = None
# remove 'additionalProperties' from tools
- schema_value: Final[JsonValue] = cast(JsonValue, value) # cast-ok: tool schemas come from JSON request data
+ schema_value: Final[JsonValue] = deepcopy(cast(JsonValue, value)) # cast-ok: tools are JSON request data
remove_additional_properties(schema_value)
# remove 'strict' from tools
remove_strict_from_schema(schema_value)
- for tool in value:
+ for tool in cast(list[dict], schema_value): # cast-ok: a deep copy keeps the caller's tool shape
openai_function_object: ChatCompletionToolParamFunctionChunk | None = None
if "function" in tool: # tools list
_openai_function_object = ChatCompletionToolParamFunctionChunk(**tool["function"])
@@ -732,7 +734,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
and _openai_function_object["parameters"] is not None
and isinstance(_openai_function_object["parameters"], dict)
): # OPENAI accepts JSON Schema, Google accepts OpenAPI schema.
- _openai_function_object["parameters"] = build_vertex_schema(_openai_function_object["parameters"])
+ _openai_function_object["parameters"] = build_vertex_schema(
+ _openai_function_object["parameters"], send_integer_enums=send_integer_enums
+ )
openai_function_object = _openai_function_object
@@ -875,21 +879,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
self,
value: list[dict[str, object]], # mutable-ok: mirrors override contract
optional_params: dict[str, object], # mutable-ok: mirrors override contract
+ send_integer_enums: bool = False,
) -> list[Tools]: # mutable-ok: mirrors override contract
- return self._map_function(value, optional_params)
+ return self._map_function(value, optional_params, send_integer_enums)
- def _map_response_schema(self, value: dict) -> dict:
+ def _map_response_schema(self, value: dict, send_integer_enums: bool = False) -> Mapping[str, object]:
old_schema = deepcopy(value)
if isinstance(old_schema, list):
for item in old_schema:
if isinstance(item, dict):
- item = build_vertex_schema(parameters=item, add_property_ordering=True)
+ item = build_vertex_schema(
+ parameters=item, add_property_ordering=True, send_integer_enums=send_integer_enums
+ )
elif isinstance(old_schema, dict):
- old_schema = build_vertex_schema(parameters=old_schema, add_property_ordering=True)
+ old_schema = build_vertex_schema(
+ parameters=old_schema, add_property_ordering=True, send_integer_enums=send_integer_enums
+ )
return old_schema
- def apply_response_schema_transformation(self, value: dict, optional_params: dict, model: str):
+ def apply_response_schema_transformation(
+ self, value: dict, optional_params: dict, model: str, allow_integer_enums: bool = True
+ ):
new_value = deepcopy(value)
# remove 'strict' from json schema (not supported by Gemini)
new_value = remove_strict_from_schema(new_value)
@@ -931,7 +942,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# - OpenAPI-style format (uppercase types)
# - No additionalProperties support
# - Requires propertyOrdering
- optional_params["response_schema"] = self._map_response_schema(value=schema)
+ optional_params["response_schema"] = with_integer_enums_that_fit(
+ partial(self._map_response_schema, schema),
+ allow_integer_enums and self._is_gemini_3_or_newer(model),
+ )
@property
def accepts_reasoning_summary_in_reasoning_effort(self) -> bool:
@@ -1249,6 +1263,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
optional_params: dict,
model: str,
drop_params: bool,
+ allow_integer_enums: bool = True,
) -> dict:
self._apply_include_server_side_tool_invocations(non_default_params, optional_params)
gemini_sampling_params_warned: bool = False
@@ -1299,7 +1314,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
elif param == "max_tokens" or param == "max_completion_tokens":
optional_params["max_output_tokens"] = value
elif param == "response_format" and isinstance(value, dict):
- self.apply_response_schema_transformation(value=value, optional_params=optional_params, model=model)
+ self.apply_response_schema_transformation(
+ value=value, optional_params=optional_params, model=model, allow_integer_enums=allow_integer_enums
+ )
elif param == "frequency_penalty":
if self._supports_penalty_parameters(model):
optional_params["frequency_penalty"] = value
@@ -1312,7 +1329,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
optional_params["logprobs"] = value
elif (param == "tools" or param == "functions") and isinstance(value, list) and value:
# Pass optional_params so _map_function can add toolConfig if needed
- mapped_tools = self._map_function(value=value, optional_params=optional_params)
+ mapped_tools = with_integer_enums_that_fit(
+ partial(self._map_function, value, optional_params),
+ allow_integer_enums and self._is_gemini_3_or_newer(model),
+ )
optional_params = self._add_tools_to_optional_params(optional_params, mapped_tools)
elif param == "tool_choice" and (isinstance(value, str) or isinstance(value, dict)):
_tool_choice_value = self.map_tool_choice_values(
diff --git a/tests/integration/providers/test_gemini_anyof_null_branches_wire.py b/tests/integration/providers/test_gemini_anyof_null_branches_wire.py
index 9fa99cb69f8..79820ac2efb 100644
--- a/tests/integration/providers/test_gemini_anyof_null_branches_wire.py
+++ b/tests/integration/providers/test_gemini_anyof_null_branches_wire.py
@@ -360,8 +360,9 @@ def test_a_repeated_identical_request_sends_the_same_folded_union_each_time(gate
model: Final = _deployment(gateway, scenario, wire, "vertex_ai")
description: Final = _description()
path, body = _request_body("chat", model, description, _parameters({"anyOf": [_NULL, _NULL, _STRING]}), False)
- first: Final = gateway.request("POST", path, body)
- second: Final = gateway.request("POST", path, body)
+ uncached: Final[dict[str, JsonValue]] = {**body, "cache": {"no-cache": True}}
+ first: Final = gateway.request("POST", path, uncached)
+ second: Final = gateway.request("POST", path, uncached)
_assert_caller_saw_ok("chat", first, False)
_assert_caller_saw_ok("chat", second, False)
assert _json(first)["id"] != _json(second)["id"]
diff --git a/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py b/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py
index b4c654ee175..da8ab138416 100644
--- a/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py
+++ b/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py
@@ -9,6 +9,7 @@ from respx import MockRouter
import litellm
from litellm.google_genai import agenerate_content_stream
from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig
+from litellm.llms.vertex_ai.common_utils import GEMINI_MAX_ENUM_CHARACTERS
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_PROMPT: Final = "Hello, can you tell me a short joke?"
@@ -239,3 +240,43 @@ async def test_generate_content_stream_parses_function_call(
},
)
assert _JSON_OBJECT.validate_json(route.calls.last.request.content)["tools"] == tools
+
+
+@pytest.mark.parametrize(
+ ("model", "expected_level"),
+ [
+ ("gemini-flash-latest", {"type": "integer", "enum": ["1", "2", "5"]}),
+ ("gemini-1.5-pro", {"type": "integer"}),
+ ],
+)
+def test_normalize_response_schema_sends_integer_enums_to_gemini_3(
+ model: str, expected_level: dict[str, object]
+) -> None:
+ config_dict: Final = {
+ "responseSchema": {"type": "object", "properties": {"level": {"type": "integer", "enum": [1, 2, 5]}}}
+ }
+
+ GoogleGenAIConfig._normalize_response_schema(config_dict, model)
+
+ assert config_dict["responseSchema"]["properties"]["level"] == expected_level
+
+
+@pytest.mark.parametrize(
+ ("value_count", "integer_enum_sent"),
+ [(GEMINI_MAX_ENUM_CHARACTERS // 10, True), (GEMINI_MAX_ENUM_CHARACTERS // 10 + 1, False)],
+)
+def test_normalize_response_schema_drops_integer_enums_past_the_enum_budget(
+ value_count: int, integer_enum_sent: bool
+) -> None:
+ values: Final = [10**9 + n for n in range(value_count)]
+ config_dict: Final = {
+ "responseSchema": {"type": "object", "properties": {"level": {"type": "integer", "enum": values}}}
+ }
+
+ GoogleGenAIConfig._normalize_response_schema(config_dict, "gemini-flash-latest")
+
+ assert config_dict["responseSchema"]["properties"]["level"] == (
+ {"type": "integer", "enum": [str(value) for value in values]}
+ if integer_enum_sent
+ else {"type": "integer"}
+ )
diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py
index 405b7234c81..948f66ab84d 100644
--- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py
+++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py
@@ -73,7 +73,7 @@ _FUNCTION_DECLARATION: Final = {
ids=("anthropic", "openai-chat", "responses"),
)
def test_function_tool_shapes_map_to_vertex_declarations(tool: JsonValue) -> None:
- context: Final = vertex_gemini_count_context(system=None, tools=[tool])
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash", system=None, tools=[tool])
assert isinstance(context, VertexGeminiCountContext)
assert context.tools == (_FUNCTION_DECLARATION,)
@@ -82,7 +82,7 @@ def test_function_tool_shapes_map_to_vertex_declarations(tool: JsonValue) -> Non
def test_long_anthropic_tool_name_is_truncated_for_vertex() -> None:
name: Final = "mcp__docs__" + "a" * 70
- context: Final = vertex_gemini_count_context(
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash",
system=None,
tools=[{"name": name, "description": "Look up a value", "input_schema": _SCHEMA}],
)
@@ -104,7 +104,7 @@ def test_long_anthropic_tool_name_is_truncated_for_vertex() -> None:
def test_64_character_anthropic_tool_name_is_unchanged_for_vertex() -> None:
name: Final = "mcp__docs__" + "a" * 53
- context: Final = vertex_gemini_count_context(
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash",
system=None,
tools=[{"name": name, "input_schema": _SCHEMA}],
)
@@ -115,7 +115,7 @@ def test_64_character_anthropic_tool_name_is_unchanged_for_vertex() -> None:
def test_long_openai_chat_tool_name_is_unchanged_for_vertex() -> None:
name: Final = "mcp__docs__" + "a" * 70
- context: Final = vertex_gemini_count_context(
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash",
system=None,
tools=[{"type": "function", "function": {"name": name, "parameters": _SCHEMA}}],
)
@@ -125,7 +125,7 @@ def test_long_openai_chat_tool_name_is_unchanged_for_vertex() -> None:
def test_unknown_tools_are_omitted() -> None:
- context: Final = vertex_gemini_count_context(
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash",
system=None,
tools=[{"type": "web_search_preview"}, {"type": "computer_use"}],
)
@@ -135,7 +135,7 @@ def test_unknown_tools_are_omitted() -> None:
def test_text_block_system_instruction_keeps_one_part_per_block() -> None:
- context: Final = vertex_gemini_count_context(
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash",
system=[{"type": "text", "text": "First instruction."}, {"type": "text", "text": "Second instruction."}],
tools=None,
)
@@ -145,14 +145,14 @@ def test_text_block_system_instruction_keeps_one_part_per_block() -> None:
def test_string_system_instruction_maps_to_one_part() -> None:
- context: Final = vertex_gemini_count_context(system="Follow this instruction.", tools=None)
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash", system="Follow this instruction.", tools=None)
assert isinstance(context, VertexGeminiCountContext)
assert context.system_instruction == {"parts": ({"text": "Follow this instruction."},)}
def test_non_text_system_block_returns_an_error_value() -> None:
- context: Final = vertex_gemini_count_context(
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash",
system=[{"type": "image", "source": {"type": "base64", "data": "AAAA"}}],
tools=None,
)
@@ -163,7 +163,7 @@ def test_non_text_system_block_returns_an_error_value() -> None:
@pytest.mark.parametrize(("system", "tools"), ((None, None), ("", None), ([], [])))
def test_empty_context_is_omitted(system: JsonValue | None, tools: JsonValue | None) -> None:
- context: Final = vertex_gemini_count_context(system=system, tools=tools)
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash", system=system, tools=tools)
assert isinstance(context, VertexGeminiCountContext)
assert context.system_instruction is None
@@ -185,7 +185,7 @@ def test_anthropic_tools_count_the_same_declarations_inference_sends(tool: dict[
optional_params={},
)
- context: Final = vertex_gemini_count_context(system=None, tools=[tool])
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash", system=None, tools=[tool])
assert isinstance(context, VertexGeminiCountContext)
assert context.tools is not None
@@ -194,6 +194,33 @@ def test_anthropic_tools_count_the_same_declarations_inference_sends(tool: dict[
]
+@pytest.mark.parametrize("model", ("gemini-3.8-flash", "gemini-2.5-flash"))
+@pytest.mark.parametrize("ids", (range(3), range(10**12, 10**12 + 200)), ids=("small-enum", "past-enum-budget"))
+def test_count_context_counts_the_integer_enums_inference_sends(model: str, ids: range) -> None:
+ tool: Final = {
+ "type": "function",
+ "function": {
+ "name": "open_record",
+ "parameters": {
+ "type": "object",
+ "properties": {"record_id": {"type": "integer", "enum": list(ids)}},
+ "required": ["record_id"],
+ },
+ },
+ }
+ inference_params: Final = VertexGeminiConfig().map_openai_params(
+ non_default_params={"tools": [tool]}, optional_params={}, model=model, drop_params=False
+ )
+
+ context: Final = vertex_gemini_count_context(model=model, system=None, tools=[tool])
+
+ assert isinstance(context, VertexGeminiCountContext)
+ assert context.tools is not None
+ assert [tool_value["functionDeclarations"] for tool_value in context.tools] == [
+ inference_tool["function_declarations"] for inference_tool in inference_params["tools"]
+ ]
+
+
@pytest.mark.parametrize(
"tools",
(
@@ -206,7 +233,7 @@ def test_anthropic_tools_count_the_same_declarations_inference_sends(tool: dict[
ids=("not-a-list", "non-object-tool", "string-schema", "list-parameters", "malformed-property"),
)
def test_invalid_tools_return_an_error_value(tools: JsonValue) -> None:
- context: Final = vertex_gemini_count_context(system=None, tools=tools)
+ context: Final = vertex_gemini_count_context(model="gemini-2.5-flash", system=None, tools=tools)
assert isinstance(context, VertexGeminiCountContextError)
assert "tools" in context.error_message
diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py
index dde3df6fcc0..dc1e762e752 100644
--- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py
+++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py
@@ -6,6 +6,7 @@ Includes tests for Vertex AI batch output transformation to OpenAI format.
import json
import urllib.parse
from collections.abc import Mapping
+from typing import Final
from types import MappingProxyType
from urllib.parse import parse_qs, urlparse
@@ -1638,6 +1639,45 @@ class TestVertexResponsesBatchInputTranslation:
assert request["contents"] == [{"role": "user", "parts": [{"text": "ping"}]}]
+_PRIORITY_SCHEMA: Final = {
+ "type": "object",
+ "properties": {
+ "level": {"type": "integer", "enum": [1, 2, 5]},
+ "mode": {"type": "string", "enum": ["fast", "slow"]},
+ },
+}
+
+
+class TestVertexBatchIntegerEnums:
+ """
+ Vertex batch reads each row's string enum values back as numbers and then fails the
+ row, so batch rows drop integer enums on every model, as before Gemini 3 sent them.
+ """
+
+ @pytest.mark.parametrize("model", ["gemini-3.8-flash", "vertex-batch-alias"])
+ def test_tool_rows_drop_integer_enums_and_keep_string_enums(self, model: str):
+ (row,) = _wrap_entries(
+ [
+ {
+ "custom_id": "enum-1",
+ "method": "POST",
+ "url": "/v1/chat/completions",
+ "body": {
+ "model": model,
+ "messages": [{"role": "user", "content": "Set priority 3"}],
+ "tools": [
+ {"type": "function", "function": {"name": "set_priority", "parameters": _PRIORITY_SCHEMA}}
+ ],
+ },
+ }
+ ]
+ )
+
+ properties: Final = row["request"]["tools"][0]["function_declarations"][0]["parameters"]["properties"]
+ assert "enum" not in properties["level"]
+ assert properties["mode"]["enum"] == ["fast", "slow"]
+
+
class TestVertexEmbeddingsBatchOutputTranslation:
"""Vertex Gemini Embedding batch output rows must come back as OpenAI batch rows."""
diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
index 5c0e9f1a305..636d4034ccc 100644
--- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
+++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
@@ -17,7 +17,7 @@ from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
-from litellm.llms.vertex_ai.common_utils import VertexAIError
+from litellm.llms.vertex_ai.common_utils import GEMINI_MAX_ENUM_CHARACTERS, VertexAIError
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
@@ -2120,6 +2120,207 @@ def test_gemini_reasoning_effort_and_function_tool_share_request() -> None:
]
+@pytest.mark.parametrize(
+ ("model", "expected_level"),
+ [
+ ("gemini-3.8-flash", {"type": "integer", "enum": ["1", "2", "5"]}),
+ ("gemini-2.5-flash", {"type": "integer"}),
+ ],
+)
+def test_gemini_tool_mapping_sends_integer_enums_to_gemini_3_and_keeps_caller_tools(
+ model: str, expected_level: dict[str, object]
+) -> None:
+ tools: Final = [
+ {
+ "type": "function",
+ "function": {
+ "name": "set_priority",
+ "strict": True,
+ "parameters": {
+ "type": "object",
+ "properties": {"level": {"type": "integer", "enum": [1, 2, 5]}},
+ "required": ["level"],
+ "additionalProperties": False,
+ },
+ },
+ }
+ ]
+ caller_tools: Final = deepcopy(tools)
+
+ request_params: Final = VertexGeminiConfig().map_openai_params(
+ non_default_params={"tools": tools},
+ optional_params={},
+ model=model,
+ drop_params=False,
+ )
+
+ sent_level: Final = request_params["tools"][0]["function_declarations"][0]["parameters"]["properties"]["level"]
+ assert sent_level == expected_level
+ assert tools == caller_tools
+
+
+@pytest.mark.parametrize(
+ ("model", "expected_level"),
+ [
+ ("gemini-flash-latest", {"type": "integer", "enum": ["1", "2", "5"]}),
+ ("gemini-1.5-pro", {"type": "integer"}),
+ ],
+)
+def test_gemini_response_schema_sends_integer_enums_to_gemini_3(model: str, expected_level: dict[str, object]) -> None:
+ response_format: Final = {
+ "type": "json_schema",
+ "json_schema": {
+ "name": "task",
+ "schema": {
+ "type": "object",
+ "properties": {"level": {"type": "integer", "enum": [1, 2, 5]}},
+ "required": ["level"],
+ },
+ },
+ }
+
+ request_params: Final = VertexGeminiConfig().map_openai_params(
+ non_default_params={"response_format": response_format},
+ optional_params={},
+ model=model,
+ drop_params=False,
+ )
+
+ assert request_params["response_schema"]["properties"]["level"] == expected_level
+
+
+def test_gemini_mapping_drops_integer_enums_when_the_caller_disallows_them() -> None:
+ level_schema: Final = {"type": "object", "properties": {"level": {"type": "integer", "enum": [1, 2, 5]}}}
+
+ request_params: Final = VertexGeminiConfig().map_openai_params(
+ non_default_params={
+ "tools": [{"type": "function", "function": {"name": "set_priority", "parameters": level_schema}}],
+ "response_format": {"type": "json_schema", "json_schema": {"name": "task", "schema": level_schema}},
+ },
+ optional_params={},
+ model="gemini-flash-latest",
+ drop_params=False,
+ allow_integer_enums=False,
+ )
+
+ assert request_params["tools"][0]["function_declarations"][0]["parameters"]["properties"]["level"] == {
+ "type": "integer"
+ }
+ assert request_params["response_schema"]["properties"]["level"] == {"type": "integer"}
+
+
+_TEN_DIGIT_VALUES_IN_BUDGET: Final = GEMINI_MAX_ENUM_CHARACTERS // 10
+
+
+def _ten_digit_integer_enum_tool(name: str, value_count: int) -> dict[str, object]:
+ return {
+ "type": "function",
+ "function": {
+ "name": name,
+ "parameters": {
+ "type": "object",
+ "properties": {"level": {"type": "integer", "enum": [10**9 + n for n in range(value_count)]}},
+ "required": ["level"],
+ },
+ },
+ }
+
+
+@pytest.mark.parametrize(
+ ("value_counts", "integer_enums_sent"),
+ [
+ ((_TEN_DIGIT_VALUES_IN_BUDGET,), True),
+ ((_TEN_DIGIT_VALUES_IN_BUDGET + 1,), False),
+ ((_TEN_DIGIT_VALUES_IN_BUDGET // 2, _TEN_DIGIT_VALUES_IN_BUDGET - _TEN_DIGIT_VALUES_IN_BUDGET // 2), True),
+ ((_TEN_DIGIT_VALUES_IN_BUDGET // 2, _TEN_DIGIT_VALUES_IN_BUDGET - _TEN_DIGIT_VALUES_IN_BUDGET // 2 + 1), False),
+ ],
+ ids=["one-tool-at-budget", "one-tool-over-budget", "two-tools-at-budget", "two-tools-over-budget"],
+)
+def test_gemini_tool_mapping_drops_every_integer_enum_when_the_request_exceeds_the_enum_budget(
+ value_counts: tuple[int, ...], integer_enums_sent: bool
+) -> None:
+ request_params: Final = VertexGeminiConfig().map_openai_params(
+ non_default_params={
+ "tools": [_ten_digit_integer_enum_tool(f"tool_{i}", count) for i, count in enumerate(value_counts)]
+ },
+ optional_params={},
+ model="gemini-3.8-flash",
+ drop_params=False,
+ )
+
+ sent_levels: Final = [
+ declaration["parameters"]["properties"]["level"]
+ for declaration in request_params["tools"][0]["function_declarations"]
+ ]
+ assert sent_levels == [
+ (
+ {"type": "integer", "enum": [str(10**9 + n) for n in range(count)]}
+ if integer_enums_sent
+ else {"type": "integer"}
+ )
+ for count in value_counts
+ ]
+
+
+def test_gemini_tool_mapping_counts_string_enums_toward_the_enum_budget() -> None:
+ names: Final = [f"name{n:06d}" for n in range(GEMINI_MAX_ENUM_CHARACTERS // 10)]
+ tool: Final = {
+ "type": "function",
+ "function": {
+ "name": "set_priority",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string", "enum": names},
+ "level": {"type": "integer", "enum": [1, 2, 5]},
+ },
+ },
+ },
+ }
+
+ request_params: Final = VertexGeminiConfig().map_openai_params(
+ non_default_params={"tools": [tool]}, optional_params={}, model="gemini-3.8-flash", drop_params=False
+ )
+
+ sent_properties: Final = request_params["tools"][0]["function_declarations"][0]["parameters"]["properties"]
+ assert sent_properties["level"] == {"type": "integer"}
+ assert sent_properties["name"]["enum"] == names
+
+
+@pytest.mark.parametrize(
+ ("value_count", "expected_level"),
+ [
+ (
+ _TEN_DIGIT_VALUES_IN_BUDGET,
+ {"type": "integer", "enum": [str(10**9 + n) for n in range(_TEN_DIGIT_VALUES_IN_BUDGET)]},
+ ),
+ (_TEN_DIGIT_VALUES_IN_BUDGET + 1, {"type": "integer"}),
+ ],
+)
+def test_gemini_response_schema_drops_integer_enums_past_the_enum_budget(
+ value_count: int, expected_level: dict[str, object]
+) -> None:
+ response_format: Final = {
+ "type": "json_schema",
+ "json_schema": {
+ "name": "task",
+ "schema": {
+ "type": "object",
+ "properties": {"level": {"type": "integer", "enum": [10**9 + n for n in range(value_count)]}},
+ },
+ },
+ }
+
+ request_params: Final = VertexGeminiConfig().map_openai_params(
+ non_default_params={"response_format": response_format},
+ optional_params={},
+ model="gemini-flash-latest",
+ drop_params=False,
+ )
+
+ assert request_params["response_schema"]["properties"]["level"] == expected_level
+
+
def test_vertex_ai_transform_parts():
"""
Test the _transform_parts method for converting Vertex AI function calls
diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py
index 38737df7545..152ab2ffc13 100644
--- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py
+++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py
@@ -2,6 +2,7 @@ import copy
import time
from collections.abc import Iterator
from contextlib import contextmanager
+from enum import Enum
from typing import Final
from unittest.mock import AsyncMock, patch
@@ -15,6 +16,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler
from litellm.llms.vertex_ai.common_utils import (
VertexAITokenCounter,
+ _fix_enum_types,
build_vertex_schema,
convert_anyof_null_to_nullable,
convert_vertex_datetime_to_openai_datetime,
@@ -1407,12 +1409,11 @@ def test_construct_target_url_versionless_project_route_gets_api_version(
def test_fix_enum_types():
"""
- Test _fix_enum_types function removes enum fields when type is not string.
+ Test _fix_enum_types keeps string enums, converts integer enums to the form
+ Gemini accepts, and removes enums on other non-string types.
- This test verifies the fix for the issue where Gemini rejects cached content
- with function parameter enums on non-string types, causing API failures.
-
- Relevant issue: Gemini only allows enums for string-typed fields
+ Gemini rejects non-string enum values, causing API failures. Integer enums are
+ sent as type INTEGER with string values.
"""
from litellm.llms.vertex_ai.common_utils import _fix_enum_types
@@ -1426,7 +1427,7 @@ def test_fix_enum_types():
"description": "How to truncate content",
},
"maxLength": {
- "enum": [100, 200, 500], # This should be removed
+ "enum": [100, 200, 500],
"type": "integer",
"description": "Maximum length",
},
@@ -1443,7 +1444,7 @@ def test_fix_enum_types():
"type": "string",
},
"innerNonStringEnum": {
- "enum": [1, 2, 3], # This should be removed
+ "enum": [1, 2, 3],
"type": "integer",
},
},
@@ -1454,13 +1455,13 @@ def test_fix_enum_types():
"type": "string",
"enum": ["option1", "option2"],
}, # This should be kept
- {"type": "integer", "enum": [1, 2, 3]}, # This should be removed
+ {"type": "integer", "enum": [1, 2, 3]},
]
},
},
}
- # Expected output: Non-string enums removed, string enums kept
+ # Expected output: string enums kept, integer enums converted, others removed
expected_output = {
"type": "object",
"properties": {
@@ -1469,7 +1470,8 @@ def test_fix_enum_types():
"type": "string",
"description": "How to truncate content",
},
- "maxLength": { # enum removed
+ "maxLength": {
+ "enum": ["100", "200", "500"],
"type": "integer",
"description": "Maximum length",
},
@@ -1484,7 +1486,10 @@ def test_fix_enum_types():
"enum": ["a", "b", "c"], # Kept - string type
"type": "string",
},
- "innerNonStringEnum": {"type": "integer"}, # enum removed
+ "innerNonStringEnum": {
+ "enum": ["1", "2", "3"],
+ "type": "integer",
+ },
},
},
"anyOfField": {
@@ -1493,14 +1498,14 @@ def test_fix_enum_types():
"type": "string",
"enum": ["option1", "option2"],
}, # Kept - has string type
- {"type": "integer"}, # enum removed
+ {"type": "integer", "enum": ["1", "2", "3"]},
]
},
},
}
# Apply the transformation
- _fix_enum_types(input_schema)
+ _fix_enum_types(input_schema, send_integer_enums=True)
# Verify the transformation
assert input_schema == expected_output
@@ -1522,20 +1527,135 @@ def test_fix_enum_types():
"c",
]
- # 2. Non-string enums are removed
- assert "enum" not in input_schema["properties"]["maxLength"]
+ # 2. Integer enums are converted, other non-string enums are removed
+ assert input_schema["properties"]["maxLength"]["enum"] == ["100", "200", "500"]
assert "enum" not in input_schema["properties"]["enabled"]
- assert "enum" not in input_schema["properties"]["nested"]["properties"]["innerNonStringEnum"]
+ assert input_schema["properties"]["nested"]["properties"]["innerNonStringEnum"]["enum"] == ["1", "2", "3"]
- # 3. anyOf with string type keeps enum, non-string removes it
+ # 3. anyOf keeps the string enum and converts the integer one
assert "enum" in input_schema["properties"]["anyOfField"]["anyOf"][0]
- assert "enum" not in input_schema["properties"]["anyOfField"]["anyOf"][1]
+ assert input_schema["properties"]["anyOfField"]["anyOf"][1]["enum"] == ["1", "2", "3"]
# 4. Other properties preserved
assert input_schema["properties"]["maxLength"]["type"] == "integer"
assert input_schema["properties"]["enabled"]["type"] == "boolean"
+
+def test_build_vertex_schema_sends_integer_enum_as_string_values():
+ """An integer enum tool parameter reaches Gemini as INTEGER with string values and no enum format."""
+ from litellm.llms.vertex_ai.common_utils import _build_vertex_schema
+
+ parameters = {
+ "type": "object",
+ "properties": {
+ "level": {"type": "integer", "enum": [1, 2, 5]},
+ "flag": {"type": "boolean", "enum": [True]},
+ },
+ "required": ["level"],
+ }
+
+ result = _build_vertex_schema(parameters, send_integer_enums=True)
+
+ assert "format" not in result["properties"]["level"]
+ assert result["properties"]["level"]["enum"] == ["1", "2", "5"]
+ assert result["properties"]["level"]["type"].lower() == "integer"
+ assert "enum" not in result["properties"]["flag"]
+
+
+@pytest.mark.parametrize(
+ ("enum_values", "send_integer_enums", "expected_level"),
+ [
+ (
+ [1, 2, 5],
+ True,
+ {"anyOf": [{"type": "integer", "enum": ["1", "2", "5"], "nullable": True}]},
+ ),
+ (
+ [1, 2, 5, None],
+ True,
+ {"anyOf": [{"type": "integer", "enum": ["1", "2", "5"], "nullable": True}]},
+ ),
+ ([1, 2, 5], False, {"anyOf": [{"type": "integer", "nullable": True}]}),
+ ([1, "a"], True, {"anyOf": [{"type": "integer", "nullable": True}]}),
+ ],
+ ids=["sent", "null-in-enum", "not-sent", "mixed-values"],
+)
+def test_build_vertex_schema_keeps_integer_enum_on_nullable_type_list(
+ enum_values: list[object], send_integer_enums: bool, expected_level: JsonValue
+) -> None:
+ schema: Final = build_vertex_schema(
+ {"type": "object", "properties": {"level": {"type": ["integer", "null"], "enum": enum_values}}},
+ send_integer_enums=send_integer_enums,
+ )
+
+ assert schema["properties"]["level"] == expected_level
+
+
+def test_map_function_keeps_integer_enum_when_tools_are_reused():
+ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
+
+ tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "set_priority",
+ "parameters": {
+ "type": "object",
+ "properties": {"level": {"type": "integer", "enum": [1, 2, 5]}},
+ "required": ["level"],
+ },
+ },
+ }
+ ]
+ config = VertexGeminiConfig()
+
+ sent = [
+ config._map_function(tools, {}, send_integer_enums=True)[0]["function_declarations"][0]["parameters"][
+ "properties"
+ ]["level"]
+ for _ in range(2)
+ ]
+
+ assert sent[0]["enum"] == ["1", "2", "5"]
+ assert sent[1]["enum"] == ["1", "2", "5"]
+ assert sent[0] == sent[1]
+
+
+class _Priority(int, Enum):
+ LOW = 1
+ HIGH = 3
+
+
+@pytest.mark.parametrize(
+ ("enum_values", "send_integer_enums", "expected_enum"),
+ [
+ ([1, 2, 5], False, None),
+ (["-3", "007", 4], True, ["-3", "7", "4"]),
+ ([1, 2**53], True, None),
+ ([1, -(2**53)], True, None),
+ (["1", str(2**53)], True, None),
+ ([-(2**53 - 1), 2**53 - 1], True, [str(-(2**53 - 1)), str(2**53 - 1)]),
+ ([str(-(2**53 - 1)), str(2**53 - 1)], True, [str(-(2**53 - 1)), str(2**53 - 1)]),
+ (["1", "9" * 5000], True, None),
+ (["1", "--2"], True, None),
+ (["1", "\u00b2"], True, None),
+ ([True, 1], True, None),
+ ([1.5, 2], True, None),
+ ([], True, None),
+ ([_Priority.LOW, _Priority.HIGH], True, ["1", "3"]),
+ ],
+)
+def test_fix_enum_types_sends_only_integer_enums_gemini_accepts(
+ enum_values: list[object], send_integer_enums: bool, expected_enum: list[str] | None
+) -> None:
+ schema: Final = {"type": "integer", "enum": enum_values}
+
+ _fix_enum_types(schema, send_integer_enums=send_integer_enums)
+
+ assert schema.get("enum") == expected_enum
+ assert "format" not in schema
+
def test_get_token_url():
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexLLM,
From 4b1928387f4197e998ad636a59e5715b4a48eb53 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 17:45:04 +0000
Subject: [PATCH 009/203] test(bedrock): detect Vault reads on the event-loop
thread (#47597)
---
.../test_bedrock_count_tokens_handler.py | 2 +-
...est_bedrock_mantle_count_tokens_handler.py | 2 +-
tests/unit/llms/bedrock/event_loop_probe.py | 20 +++++++------------
.../realtime/test_bedrock_realtime_handler.py | 2 +-
.../bedrock/test_claude_platform_provider.py | 2 +-
.../custom_httpx/test_llm_http_handler.py | 2 +-
.../test_bedrock_guardrails.py | 2 +-
.../test_llm_pass_through_endpoints.py | 2 +-
8 files changed, 14 insertions(+), 20 deletions(-)
diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py
index 5dadeff6d7c..591205b4602 100644
--- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py
+++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py
@@ -81,7 +81,7 @@ async def test_handle_count_tokens_request_looks_up_its_region_off_the_event_loo
)
assert vault.read.is_set()
- assert not vault.stalled_the_loop.is_set()
+ assert not vault.read_on_loop.is_set()
assert result == {"input_tokens": 7}
diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_mantle_count_tokens_handler.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_mantle_count_tokens_handler.py
index 2d21a3ddef5..a6b309c1b83 100644
--- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_mantle_count_tokens_handler.py
+++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_mantle_count_tokens_handler.py
@@ -124,6 +124,6 @@ async def test_region_and_endpoint_lookups_leave_the_event_loop_free(monkeypatch
)
assert vault.read.is_set()
- assert not vault.stalled_the_loop.is_set()
+ assert not vault.read_on_loop.is_set()
assert result == {"input_tokens": 2177}
assert str(mantle.only_request().url) == MANTLE_COUNT_URL
diff --git a/tests/unit/llms/bedrock/event_loop_probe.py b/tests/unit/llms/bedrock/event_loop_probe.py
index cf6cd5975ed..8bf856582b5 100644
--- a/tests/unit/llms/bedrock/event_loop_probe.py
+++ b/tests/unit/llms/bedrock/event_loop_probe.py
@@ -16,7 +16,6 @@ from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagem
REFRESH_RELEASE_TIMEOUT_SECONDS: Final = 2.0
REFRESH_START_TIMEOUT_SECONDS: Final = 10.0
-LOOP_SERVE_TIMEOUT_SECONDS: Final = 2.0
class EventLoopProbe:
@@ -63,26 +62,21 @@ class EventLoopProbe:
class LoopProbingVault:
- """A Vault holding none of its hosted secrets, whose read waits for the event loop to run a callback.
+ """A Vault with no hosted secrets that records reads on the event-loop thread."""
- A read made on the loop thread can never see that callback, so `stalled_the_loop` is set only then.
- """
-
- def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
- self._loop: Final = loop
+ def __init__(self) -> None:
+ self._loop_thread_id: Final = threading.get_ident()
self.read: Final = threading.Event()
- self.stalled_the_loop: Final = threading.Event()
+ self.read_on_loop: Final = threading.Event()
def sync_read_secret(self, secret_name: str) -> None:
self.read.set()
- served: Final = threading.Event()
- self._loop.call_soon_threadsafe(served.set)
- if not served.wait(timeout=LOOP_SERVE_TIMEOUT_SECONDS):
- self.stalled_the_loop.set()
+ if threading.get_ident() == self._loop_thread_id:
+ self.read_on_loop.set()
def host_in_loop_probing_vault(monkeypatch: pytest.MonkeyPatch, *secret_names: str) -> LoopProbingVault:
- vault: Final = LoopProbingVault(asyncio.get_running_loop())
+ vault: Final = LoopProbingVault()
monkeypatch.setattr(litellm, "secret_manager_client", vault)
monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.HASHICORP_VAULT)
monkeypatch.setattr(
diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py
index ccea4ba44d7..2ac31b6887d 100644
--- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py
+++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py
@@ -643,7 +643,7 @@ class TestBedrockRealtimeSessionLifecycle:
)
assert vault.read.is_set()
- assert not vault.stalled_the_loop.is_set()
+ assert not vault.read_on_loop.is_set()
assert stub_aws_sdk_client["config_kwargs"]["region"] == "us-west-2"
assert stub_aws_sdk_client["config_kwargs"]["endpoint_uri"] == "https://bedrock-runtime.us-west-2.amazonaws.com"
diff --git a/tests/unit/llms/bedrock/test_claude_platform_provider.py b/tests/unit/llms/bedrock/test_claude_platform_provider.py
index e43184df50e..2353a45139b 100644
--- a/tests/unit/llms/bedrock/test_claude_platform_provider.py
+++ b/tests/unit/llms/bedrock/test_claude_platform_provider.py
@@ -705,7 +705,7 @@ async def test_anthropic_messages_claude_platform_workspace_and_region_lookups_l
)
assert vault.read.is_set()
- assert not vault.stalled_the_loop.is_set()
+ assert not vault.read_on_loop.is_set()
assert response["content"][0]["text"] == "ok"
_assert_gateway_accepted(requests[0], sigv4_without_region)
diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py
index 205c610a413..63cc45b29df 100644
--- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py
+++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py
@@ -1392,7 +1392,7 @@ async def test_bedrock_secret_lookups_leave_the_event_loop_free(
_ = await call()
assert vault.read.is_set()
- assert not vault.stalled_the_loop.is_set()
+ assert not vault.read_on_loop.is_set()
assert ".us-west-2." in upstream.calls.last.request.url.host
diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py
index 75d713028d4..69b3e113b8f 100644
--- a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py
+++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py
@@ -8103,5 +8103,5 @@ async def test_guardrail_secret_lookups_leave_the_event_loop_free(
)
assert vault.read.is_set()
- assert not vault.stalled_the_loop.is_set()
+ assert not vault.read_on_loop.is_set()
assert upstream.calls.last.request.url.host == "bedrock-runtime.us-west-2.amazonaws.com"
diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
index 66c52a76318..0186a33dbc4 100644
--- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
+++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
@@ -2558,7 +2558,7 @@ async def test_bedrock_agent_runtime_region_lookup_leaves_the_event_loop_free(mo
)
assert vault.read.is_set()
- assert not vault.stalled_the_loop.is_set()
+ assert not vault.read_on_loop.is_set()
assert retrieve.calls.last.request.headers["authorization"].startswith("AWS4-HMAC-SHA256")
From b245cc0d269e880e75e4bc070045f8343f42f114 Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 10:49:02 -0700
Subject: [PATCH 010/203] feat(proxy): size-triggered spend log cleanup, oldest
rows first (#47450)
Adds general_settings.maximum_spend_logs_size,
maximum_spend_logs_cleanup_target_size and
maximum_spend_logs_size_check_interval, the spend_log_size_check job,
maximum_spend_logs_cleanup_cron precedence, oldest-first bounded deletes
down to the target, storage fields in the run summary and the
litellm_spend_log_storage_bytes gauge, with a Postgres integration test
covering the post-VACUUM no-refire case
Resolves LIT-9831
Fixes #44589
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
litellm/constants.py | 3 +
litellm/proxy/_types.py | 24 ++
.../db_transaction_queue/spend_log_cleanup.py | 388 +++++++++++++++++-
.../spend_log_cleanup_metrics.py | 19 +
litellm/proxy/proxy_server.py | 173 +++++---
.../database/test_cleanup_transactions.py | 163 ++++++++
tests/unit/proxy/test_spend_log_cleanup.py | 252 +++++++++++-
ui/litellm-dashboard/src/lib/http/schema.d.ts | 15 +
8 files changed, 962 insertions(+), 75 deletions(-)
diff --git a/litellm/constants.py b/litellm/constants.py
index bc94760b187..377baa25000 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1855,6 +1855,9 @@ SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = get_env_float(
SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", 300.0)
SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", 30.0)
SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = get_env_int("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", 100000)
+SPEND_LOG_SIZE_CHECK_JOB_NAME: Final = "spend_log_size_check"
+SPEND_LOG_SIZE_CHECK_INTERVAL: Final = os.getenv("SPEND_LOG_SIZE_CHECK_INTERVAL", "1d")
+SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES: Final = get_env_int("SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES", 10)
SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = get_env_float(
"SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", 5.0
)
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 523f478d771..315533d6d0f 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -2884,6 +2884,30 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
"deleted. Only historical tag usage analytics are affected; tag budgets read the lifetime counter."
),
)
+ maximum_spend_logs_size: str | None = Field(
+ None,
+ description=(
+ "Size rule for spend logs (e.g. '180GB'): when LiteLLM_SpendLogs plus LiteLLM_SpendLogToolIndex "
+ "(heap, indexes and TOAST, measured with pg_total_relation_size) hold more live data than this, the "
+ "cleanup job deletes the oldest spend logs first until they are under "
+ "maximum_spend_logs_cleanup_target_size. Independent of maximum_spend_logs_retention_period: a row is "
+ "deleted when either rule says so. Unset means no size rule."
+ ),
+ )
+ maximum_spend_logs_cleanup_target_size: str | None = Field(
+ None,
+ description=(
+ "Level a size-triggered cleanup run deletes down to (e.g. '20GB'). Must be below maximum_spend_logs_size. "
+ "Unset means the run stops as soon as the tables are back under maximum_spend_logs_size."
+ ),
+ )
+ maximum_spend_logs_size_check_interval: str | None = Field(
+ None,
+ description=(
+ "How often the size rule is checked (e.g. '10m'). Defaults to '1d'. Ignored when "
+ "maximum_spend_logs_cleanup_cron is set, which then drives both the age and the size cleanup."
+ ),
+ )
use_spend_logs_partitioning: bool | None = Field(
None,
description="If True and LiteLLM_SpendLogs has been converted to a range-partitioned table (db_scripts/partition_spend_logs.sql), retention cleanup drops expired partitions instead of deleting rows, and pre-creates upcoming partitions. Default is False.",
diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
index b57952b4992..268f0230d7a 100644
--- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
+++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
@@ -1,6 +1,8 @@
import asyncio
+import math
import re
import time
+from collections.abc import Callable
from contextvars import ContextVar
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
@@ -19,6 +21,7 @@ from litellm.constants import (
SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP,
SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS,
SPEND_LOG_RUN_LOOPS,
+ SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES,
)
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.proxy.db.db_span import db_span
@@ -35,7 +38,7 @@ from litellm.proxy.db.db_transaction_queue.spend_logs_partition_manager import (
from litellm.proxy.utils import PrismaClient
from litellm.types.llms.base import LiteLLMBaseModel
-StopReason: TypeAlias = Literal["exhausted", "budget_exhausted", "batch_cap_reached", "aborted"]
+StopReason: TypeAlias = Literal["exhausted", "target_reached", "budget_exhausted", "batch_cap_reached", "aborted"]
_SIZE_RE: Final = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([KMGT]?)I?B?\s*$", re.IGNORECASE)
_SIZE_UNITS: Final = {"": 1, "K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}
@@ -61,6 +64,14 @@ def size_in_bytes(value: str | int) -> int:
return size
+@dataclass(frozen=True, slots=True)
+class SizeRule:
+ """The size cleanup rule as configured: start cleaning above `limit_bytes`, stop under `target_bytes`."""
+
+ limit_bytes: int
+ target_bytes: int
+
+
@dataclass(frozen=True, slots=True)
class StorageProbe:
"""
@@ -218,6 +229,42 @@ class TableCleanupResult:
stop_reason: StopReason
+@dataclass(frozen=True, slots=True)
+class _SizeStepState:
+ """Progress of the size pass across its re-check steps."""
+
+ logs_deleted: int
+ index_deleted: int
+ batches_left: int
+ stop_reason: StopReason | None
+
+ def stopped(self, reason: StopReason) -> "_SizeStepState":
+ return _SizeStepState(
+ logs_deleted=self.logs_deleted,
+ index_deleted=self.index_deleted,
+ batches_left=self.batches_left,
+ stop_reason=reason,
+ )
+
+
+@dataclass(frozen=True, slots=True)
+class SizePassReport:
+ """What one size pass measured and achieved, for the run summary and the storage gauges."""
+
+ rule: SizeRule
+ before: SpendLogStorage
+ after: SpendLogStorage
+ results: tuple[TableCleanupResult, ...]
+
+ @property
+ def target_reached(self) -> bool:
+ return self.after.estimated_live_bytes <= self.rule.target_bytes
+
+ @property
+ def above_target_bytes(self) -> int:
+ return max(0, self.after.estimated_live_bytes - self.rule.target_bytes)
+
+
class _RunProgress:
"""How far one cleanup run has got, reported if that run is cancelled"""
@@ -277,6 +324,13 @@ _SAVE_SIZE_STATE_SQL: Final = """
"""
+class _OffsetRow(LiteLLMBaseModel):
+ model_config = ConfigDict(frozen=True)
+ startTime: datetime
+
+
+_OFFSET_ROWS: Final = TypeAdapter(list[_OffsetRow])
+
_SPEND_LOG_TABLES: Final = ("LiteLLM_SpendLogs", "LiteLLM_SpendLogToolIndex")
# Partition children are counted with their parent: a partitioned parent has no storage of its own.
@@ -456,6 +510,53 @@ class SpendLogCleanup:
self.retention_seconds = self._retention_seconds_for("maximum_spend_logs_retention_period")
return self.retention_seconds is not None
+ def _setting(self, setting_name: str) -> object:
+ return self.general_settings.get(setting_name) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped settings dict
+
+ def _size_bytes_setting(self, setting_name: str) -> int | None:
+ """Parse one size setting into bytes, or None when unset or unusable."""
+ raw: Final = self._setting(setting_name)
+ if raw is None:
+ return None
+ if not isinstance(raw, (str, int)):
+ verbose_proxy_logger.warning("Invalid %s value: %r, ignoring it", setting_name, raw)
+ return None
+ try:
+ return size_in_bytes(raw)
+ except ValueError as e:
+ verbose_proxy_logger.warning("Invalid %s value: %r (%s), ignoring it", setting_name, raw, e)
+ return None
+
+ def _size_rule(self) -> SizeRule | None:
+ """
+ The size rule from settings, or None when maximum_spend_logs_size is unset.
+
+ A target at or above the limit cannot be what the operator meant, so it is reported and the run
+ cleans only until it is back under the limit, which is also what an unset target means.
+ """
+ limit_bytes: Final = self._size_bytes_setting("maximum_spend_logs_size")
+ if limit_bytes is None:
+ return None
+ target_bytes: Final = self._size_bytes_setting("maximum_spend_logs_cleanup_target_size")
+ if target_bytes is None:
+ return SizeRule(limit_bytes=limit_bytes, target_bytes=limit_bytes)
+ if target_bytes >= limit_bytes:
+ verbose_proxy_logger.error(
+ "maximum_spend_logs_cleanup_target_size (%s bytes) must be below maximum_spend_logs_size (%s bytes); "
+ "cleaning down to maximum_spend_logs_size instead",
+ target_bytes,
+ limit_bytes,
+ )
+ return SizeRule(limit_bytes=limit_bytes, target_bytes=limit_bytes)
+ return SizeRule(limit_bytes=limit_bytes, target_bytes=target_bytes)
+
+ def _size_pass_runs_with_age_pass(self) -> bool:
+ """
+ With maximum_spend_logs_cleanup_cron set there is one schedule for both rules, so the age run carries
+ the size pass; otherwise the size pass has its own job on maximum_spend_logs_size_check_interval.
+ """
+ return bool(self._setting("maximum_spend_logs_cleanup_cron"))
+
async def _probe_storage(self, prisma_client: PrismaClient, deadline: float) -> StorageProbe | None:
"""Read what the spend-log tables weigh from the catalog, or None when the probe fails or has no budget."""
if time.monotonic() >= deadline:
@@ -508,6 +609,32 @@ class SpendLogCleanup:
except Exception as e: # noqa: BLE001 - a lost save costs one re-seed from the catalog, it must not fail the run
verbose_proxy_logger.warning("Could not save the spend-log size cleanup state: %s: %s", type(e).__name__, e)
+ async def _size_step_cutoff(self, prisma_client: PrismaClient, deadline: float, rows: int) -> datetime | None:
+ """
+ The startTime that puts the `rows` oldest rows behind it.
+
+ Expressing a size step as a cutoff lets the size pass reuse the retention DELETE unchanged, including
+ the tool-index rows that expire with their spend logs. When fewer rows than that remain, the step is
+ everything up to now.
+ """
+ if time.monotonic() >= deadline:
+ return None
+ timeout_ms: Final = self._timeout_ms(deadline)
+ async with db_span("size_step_cutoff", "LiteLLM_SpendLogs"), bounded_tx(prisma_client, timeout_ms) as tx:
+ await tx.execute_raw(trusted_sql(f"SET LOCAL statement_timeout = {timeout_ms}"))
+ found: Final = _OFFSET_ROWS.validate_python(
+ await tx.query_raw(
+ trusted_sql(
+ 'SELECT "startTime" FROM "LiteLLM_SpendLogs" ORDER BY "startTime" ASC OFFSET $1 LIMIT 1'
+ ),
+ rows,
+ )
+ )
+ if not found:
+ return datetime.now(timezone.utc)
+ start_time: Final = found[0].startTime
+ return start_time if start_time.tzinfo is not None else start_time.replace(tzinfo=timezone.utc)
+
def _timeout_ms(self, deadline: float) -> int:
"""
The per-statement bound in milliseconds: the batch timeout, or whatever
@@ -621,14 +748,17 @@ class SpendLogCleanup:
key_columns: tuple[str, ...],
time_column: str,
deadline: float,
+ batch_cap: int | None = None,
) -> TableCleanupResult:
"""
Delete a table's rows older than the cutoff in batches.
Stops at whichever bound is reached first: the backlog running out, the
- shared wall-clock deadline, the per-table batch cap, or too many
- consecutive batch failures.
+ shared wall-clock deadline, the per-table batch cap (`batch_cap` when a
+ caller spreads one table's cap over several calls, else the configured
+ one), or too many consecutive batch failures.
"""
+ max_batches: Final = self.max_batches if batch_cap is None else batch_cap
key_list: Final = ", ".join(f'"{col}"' for col in key_columns)
delete_sql: Final = f"""
DELETE FROM "{table_name}"
@@ -651,7 +781,7 @@ class SpendLogCleanup:
return await self._finish_table(
prisma_client, cutoff_date, table_name, time_column, total_deleted, "budget_exhausted", deadline
)
- if run_count >= self.max_batches:
+ if run_count >= max_batches:
verbose_proxy_logger.info(
"Max batches reached for %s cleanup, remaining rows will be deleted in next run", table_name
)
@@ -891,6 +1021,163 @@ class SpendLogCleanup:
verbose_proxy_logger.info("Deleted %s expired tool index rows", index_result.rows_deleted)
return (logs_result, index_result)
+ async def _clean_spend_logs_by_size(
+ self, prisma_client: PrismaClient, rule: SizeRule, deadline: float
+ ) -> SizePassReport | None:
+ """
+ Bring the spend-log tables under the size target, oldest rows first.
+
+ Live bytes are the files on disk less the reclaimable count carried in LiteLLM_Config (see
+ `SizeCleanupState`), rolled forward from the catalog's cumulative insert and delete counters. Rows go in steps: the rows still needed to reach the target follow
+ from bytes per row, and each step turns that many of the oldest rows (at most
+ SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES batches) into a cutoff and runs the same bounded DELETE the age
+ rule uses, for the spend logs and the tool index rows derived from them. Progress is counted from the
+ deletes themselves, not re-read from pg_stat_user_tables, whose counters trail a DELETE by up to a
+ second. The run's batch cap and budget apply to the whole pass, so a pass that stops short reports how
+ far above the target it still is and the next check continues from there.
+ """
+ probe: Final = await self._probe_storage(prisma_client, deadline)
+ if probe is None:
+ return None
+ if probe.stats_are_stale:
+ verbose_proxy_logger.warning(
+ "Spend-log row statistics are stale (allocated=%s bytes, live_rows=0, dead_rows=0); skipping the "
+ 'size pass until pg_stat_user_tables is repopulated, ANALYZE "LiteLLM_SpendLogs" refreshes it',
+ probe.allocated_bytes,
+ )
+ return None
+ state: Final = advance_size_state(await self._load_size_state(prisma_client, deadline), probe)
+ before: Final = SpendLogStorage(
+ allocated_bytes=probe.allocated_bytes,
+ reclaimable_bytes=state.reclaimable_bytes,
+ bytes_per_row=state.bytes_per_row,
+ live_rows=probe.live_rows,
+ )
+ SpendLogCleanupMetrics.set_storage_bytes("allocated", before.allocated_bytes)
+ SpendLogCleanupMetrics.set_storage_bytes("estimated_live", before.estimated_live_bytes)
+ if before.estimated_live_bytes <= rule.limit_bytes:
+ verbose_proxy_logger.info(
+ "Spend-log storage is under maximum_spend_logs_size (estimated_live=%s allocated=%s reclaimable=%s "
+ "limit=%s bytes)",
+ before.estimated_live_bytes,
+ before.allocated_bytes,
+ before.reclaimable_bytes,
+ rule.limit_bytes,
+ )
+ SpendLogCleanupMetrics.set_storage_bytes("above_target", 0)
+ await self._save_size_state(prisma_client, deadline, state)
+ return SizePassReport(rule=rule, before=before, after=before, results=())
+ verbose_proxy_logger.warning(
+ "Spend-log storage is over maximum_spend_logs_size (estimated_live=%s allocated=%s reclaimable=%s "
+ "limit=%s bytes); deleting the oldest spend logs down to %s bytes",
+ before.estimated_live_bytes,
+ before.allocated_bytes,
+ before.reclaimable_bytes,
+ rule.limit_bytes,
+ rule.target_bytes,
+ )
+ rows_to_delete: Final = max(
+ 0, math.ceil((before.estimated_live_bytes - rule.target_bytes) / before.bytes_per_row)
+ )
+
+ def rows_still_needed(rows_deleted: int) -> int:
+ return rows_to_delete - rows_deleted
+
+ results: Final = await self._delete_oldest_rows_for_size(prisma_client, deadline, rows_still_needed)
+ rows_deleted: Final = sum(result.rows_deleted for result in results)
+ remeasured: Final = await self._probe_storage(prisma_client, deadline)
+ after: Final = before.after_deleting(
+ rows_deleted, remeasured.allocated_bytes if remeasured is not None else before.allocated_bytes
+ )
+ await self._save_size_state(
+ prisma_client,
+ deadline,
+ after.to_state(inserted_total=probe.inserted_total, deleted_total=probe.deleted_total + rows_deleted),
+ )
+ report: Final = SizePassReport(rule=rule, before=before, after=after, results=results)
+ SpendLogCleanupMetrics.set_storage_bytes("allocated", after.allocated_bytes)
+ SpendLogCleanupMetrics.set_storage_bytes("estimated_live", after.estimated_live_bytes)
+ SpendLogCleanupMetrics.set_storage_bytes("above_target", report.above_target_bytes)
+ return report
+
+ async def _delete_oldest_rows_for_size(
+ self,
+ prisma_client: PrismaClient,
+ deadline: float,
+ rows_still_needed: Callable[[int], int],
+ ) -> tuple[TableCleanupResult, ...]:
+ state = _SizeStepState( # rebind-ok: one value threaded through the bounded step loop
+ logs_deleted=0, index_deleted=0, batches_left=self.max_batches, stop_reason=None
+ )
+ for _ in range(self.max_batches // SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES + 2):
+ if state.stop_reason is not None:
+ break
+ state = await self._run_size_step(prisma_client, deadline, rows_still_needed, state)
+ final_reason: Final[StopReason] = state.stop_reason or "batch_cap_reached"
+ return (
+ TableCleanupResult(
+ table_name="LiteLLM_SpendLogs (size rule)", rows_deleted=state.logs_deleted, stop_reason=final_reason
+ ),
+ TableCleanupResult(
+ table_name="LiteLLM_SpendLogToolIndex (size rule)",
+ rows_deleted=state.index_deleted,
+ stop_reason=final_reason,
+ ),
+ )
+
+ async def _run_size_step(
+ self,
+ prisma_client: PrismaClient,
+ deadline: float,
+ rows_still_needed: Callable[[int], int],
+ state: "_SizeStepState",
+ ) -> "_SizeStepState":
+ """One step: delete the rows still needed, at most SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES batches of the oldest."""
+ needed: Final = rows_still_needed(state.logs_deleted + state.index_deleted)
+ if needed <= 0:
+ return state.stopped("target_reached")
+ if time.monotonic() >= deadline:
+ return state.stopped("budget_exhausted")
+ if state.batches_left <= 0:
+ return state.stopped("batch_cap_reached")
+ step_cap: Final = min(
+ SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES, state.batches_left, math.ceil(needed / self.batch_size)
+ )
+ cutoff: Final = await self._size_step_cutoff(prisma_client, deadline, step_cap * self.batch_size)
+ if cutoff is None:
+ return state.stopped("budget_exhausted")
+ logs_result: Final = await self._delete_old_rows_batched(
+ prisma_client,
+ cutoff,
+ table_name="LiteLLM_SpendLogs",
+ key_columns=("request_id", "startTime"),
+ time_column="startTime",
+ deadline=deadline,
+ batch_cap=step_cap,
+ )
+ index_result: Final = await self._delete_old_rows_batched(
+ prisma_client,
+ cutoff,
+ table_name="LiteLLM_SpendLogToolIndex",
+ key_columns=("request_id", "tool_name"),
+ time_column="start_time",
+ deadline=deadline,
+ batch_cap=step_cap,
+ )
+ advanced: Final = _SizeStepState(
+ logs_deleted=state.logs_deleted + logs_result.rows_deleted,
+ index_deleted=state.index_deleted + index_result.rows_deleted,
+ batches_left=state.batches_left - step_cap,
+ stop_reason=None,
+ )
+ if rows_still_needed(advanced.logs_deleted + advanced.index_deleted) <= 0:
+ return advanced.stopped("target_reached")
+ if logs_result.stop_reason in ("aborted", "budget_exhausted") or index_result.stop_reason == "aborted":
+ return advanced.stopped(logs_result.stop_reason if logs_result.stop_reason != "exhausted" else "aborted")
+ if logs_result.rows_deleted == 0:
+ return advanced.stopped("exhausted")
+ return advanced
+
async def _clean_session_rollup(
self, prisma_client: PrismaClient, retention_seconds: int, deadline: float
) -> tuple[TableCleanupResult, ...]:
@@ -968,22 +1255,60 @@ class SpendLogCleanup:
return "completed"
@staticmethod
- def _log_run_summary(outcome: RunOutcome, results: tuple[TableCleanupResult, ...], elapsed_seconds: float) -> None:
+ def _storage_summary(report: SizePassReport | None) -> str:
+ """
+ The size pass in one clause: what was measured, where it ended against the target, and whether the
+ operator still has to act to get the disk back.
+ """
+ if report is None:
+ return ""
+ rows_deleted: Final = sum(result.rows_deleted for result in report.results)
+ reclaim_note: Final = (
+ "; allocated bytes stay until autovacuum reuses them, only VACUUM FULL or pg_repack on the spend-log "
+ "tables returns them to the filesystem"
+ if rows_deleted > 0 and report.after.allocated_bytes >= report.before.allocated_bytes
+ else ""
+ )
+ return (
+ f" storage: allocated_before={report.before.allocated_bytes} "
+ f"estimated_live_before={report.before.estimated_live_bytes} "
+ f"allocated_after={report.after.allocated_bytes} estimated_live_after={report.after.estimated_live_bytes} "
+ f"reclaimable_after={report.after.reclaimable_bytes} "
+ f"limit={report.rule.limit_bytes} target={report.rule.target_bytes} "
+ f"target_reached={str(report.target_reached).lower()} above_target={report.above_target_bytes}{reclaim_note}"
+ )
+
+ @classmethod
+ def _log_run_summary(
+ cls,
+ outcome: RunOutcome,
+ results: tuple[TableCleanupResult, ...],
+ elapsed_seconds: float,
+ size_report: SizePassReport | None = None,
+ ) -> None:
per_table: Final = ", ".join(
f"{result.table_name}: deleted={result.rows_deleted} stop_reason={result.stop_reason}" for result in results
)
- message: Final = "Spend log cleanup run finished: outcome=%s elapsed=%.1fs [%s]"
- if outcome == "completed":
- verbose_proxy_logger.info(message, outcome, elapsed_seconds, per_table)
+ message: Final = "Spend log cleanup run finished: outcome=%s elapsed=%.1fs [%s]%s"
+ storage: Final = cls._storage_summary(size_report)
+ if outcome == "completed" and (size_report is None or size_report.target_reached):
+ verbose_proxy_logger.info(message, outcome, elapsed_seconds, per_table, storage)
return
- verbose_proxy_logger.warning(message, outcome, elapsed_seconds, per_table)
+ verbose_proxy_logger.warning(message, outcome, elapsed_seconds, per_table, storage)
async def cleanup_old_spend_logs(self, prisma_client: PrismaClient) -> None:
"""
- Main cleanup function. Deletes old spend logs in batches.
+ The scheduled retention run: every age rule, plus the size rule when one cron drives both.
If pod_lock_manager is available, ensures only one pod runs cleanup.
If no pod_lock_manager, runs cleanup without distributed locking.
"""
+ await self._run(prisma_client, age_rules=True, size_rule=self._size_pass_runs_with_age_pass())
+
+ async def cleanup_spend_logs_by_size(self, prisma_client: PrismaClient) -> None:
+ """The size-check run scheduled on maximum_spend_logs_size_check_interval: the size rule only."""
+ await self._run(prisma_client, age_rules=False, size_rule=True)
+
+ async def _run(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> None:
lock_acquired = False
run_started_at: Final = time.monotonic()
progress: Final = _RunProgress()
@@ -992,19 +1317,23 @@ class SpendLogCleanup:
verbose_proxy_logger.info("Cleanup job triggered at %s", datetime.now())
self._refresh_bounds()
- delete_spend_logs: Final = self._should_delete_spend_logs()
- autorouter_retention_seconds: Final = self._retention_seconds_for(
- "maximum_autorouter_session_retention_period"
+ delete_spend_logs: Final = age_rules and self._should_delete_spend_logs()
+ active_size_rule: Final = self._size_rule() if size_rule else None
+ autorouter_retention_seconds: Final = (
+ self._retention_seconds_for("maximum_autorouter_session_retention_period") if age_rules else None
)
- health_check_retention_seconds: Final = self._retention_seconds_for("maximum_health_check_retention_period")
- daily_tag_spend_retention_seconds: Final = self._retention_seconds_for(
- "maximum_daily_tag_spend_retention_period"
+ health_check_retention_seconds: Final = (
+ self._retention_seconds_for("maximum_health_check_retention_period") if age_rules else None
+ )
+ daily_tag_spend_retention_seconds: Final = (
+ self._retention_seconds_for("maximum_daily_tag_spend_retention_period") if age_rules else None
)
if (
not delete_spend_logs
and autorouter_retention_seconds is None
and health_check_retention_seconds is None
and daily_tag_spend_retention_seconds is None
+ and active_size_rule is None
):
SpendLogCleanupMetrics.record_run("skipped_disabled")
return
@@ -1032,11 +1361,13 @@ class SpendLogCleanup:
return
deadline: Final = time.monotonic() + self.run_budget_seconds
+ size_group: Final = int(active_size_rule is not None)
configured_group_count: Final = (
int(delete_spend_logs and self.retention_seconds is not None)
+ int(autorouter_retention_seconds is not None)
+ int(health_check_retention_seconds is not None)
+ int(daily_tag_spend_retention_seconds is not None)
+ + size_group
)
spend_log_results: Final = (
@@ -1051,9 +1382,12 @@ class SpendLogCleanup:
int(autorouter_retention_seconds is not None)
+ int(health_check_retention_seconds is not None)
+ int(daily_tag_spend_retention_seconds is not None)
+ + size_group
)
- remaining_groups_after_sessions: Final = int(health_check_retention_seconds is not None) + int(
- daily_tag_spend_retention_seconds is not None
+ remaining_groups_after_sessions: Final = (
+ int(health_check_retention_seconds is not None)
+ + int(daily_tag_spend_retention_seconds is not None)
+ + size_group
)
session_results: Final = (
await self._clean_session_rollup(
@@ -1074,15 +1408,27 @@ class SpendLogCleanup:
else ()
)
daily_tag_spend_results: Final = (
- await self._clean_daily_tag_spend(prisma_client, daily_tag_spend_retention_seconds, deadline)
+ await self._clean_daily_tag_spend(
+ prisma_client,
+ daily_tag_spend_retention_seconds,
+ self._group_deadline(deadline, 1 + size_group),
+ )
if daily_tag_spend_retention_seconds is not None
else ()
)
+ size_report: Final = (
+ await self._clean_spend_logs_by_size(prisma_client, active_size_rule, deadline)
+ if active_size_rule is not None
+ else None
+ )
+ size_results: Final = size_report.results if size_report is not None else ()
- results: Final = spend_log_results + session_results + health_check_results + daily_tag_spend_results
+ results: Final = (
+ spend_log_results + session_results + health_check_results + daily_tag_spend_results + size_results
+ )
outcome: Final = self._run_outcome(results)
SpendLogCleanupMetrics.record_run(outcome)
- self._log_run_summary(outcome, results, time.monotonic() - run_started_at)
+ self._log_run_summary(outcome, results, time.monotonic() - run_started_at, size_report)
except asyncio.CancelledError:
verbose_proxy_logger.error(
diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup_metrics.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup_metrics.py
index 340aeab938c..048fa8127a4 100644
--- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup_metrics.py
+++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup_metrics.py
@@ -32,6 +32,9 @@ RunOutcome: TypeAlias = Literal[
_BATCH_DURATION_BUCKETS: Final = (0.005, 0.025, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0)
_TABLE_LABEL: Final = ("table",)
_OUTCOME_LABEL: Final = ("outcome",)
+_MEASURE_LABEL: Final = ("measure",)
+
+StorageMeasure: TypeAlias = Literal["allocated", "estimated_live", "above_target"]
class SpendLogCleanupMetrics:
@@ -49,6 +52,7 @@ class SpendLogCleanupMetrics:
rows_remaining: "PrometheusGauge | None" = None
batch_failures: "PrometheusCounter | None" = None
runs: "PrometheusCounter | None" = None
+ storage_bytes: "PrometheusGauge | None" = None
@classmethod
def _ensure_initialized(cls) -> None:
@@ -90,6 +94,15 @@ class SpendLogCleanupMetrics:
"Retention cleanup runs, labelled by why the run ended",
labelnames=_OUTCOME_LABEL,
)
+ cls.storage_bytes = Gauge(
+ "litellm_spend_log_storage_bytes",
+ "Disk held by LiteLLM_SpendLogs plus LiteLLM_SpendLogToolIndex as last measured by the cleanup "
+ "job: allocated is pg_total_relation_size (heap, indexes, TOAST), estimated_live scales it by the "
+ "live share of rows from pg_stat_user_tables, above_target is estimated_live minus the size "
+ "cleanup target (0 once the target is reached)",
+ labelnames=_MEASURE_LABEL,
+ multiprocess_mode="livemax",
+ )
except Exception as e: # noqa: BLE001 - a metrics problem must never fail the cleanup run
# Covers the extra being absent, a duplicate registration (repeated
# imports under a test runner), and registry misconfiguration alike.
@@ -120,3 +133,9 @@ class SpendLogCleanupMetrics:
cls._ensure_initialized()
if cls.runs is not None:
cls.runs.labels(outcome=outcome).inc()
+
+ @classmethod
+ def set_storage_bytes(cls, measure: StorageMeasure, value: int) -> None:
+ cls._ensure_initialized()
+ if cls.storage_bytes is not None:
+ cls.storage_bytes.labels(measure=measure).set(value)
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 0af6036efa7..c6d3e12a378 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -340,6 +340,7 @@ from litellm.constants import (
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG,
SPEND_CAPTURE_RATE_CHECK_JOB_ID,
+ SPEND_LOG_SIZE_CHECK_INTERVAL,
USER_SPEND_ALERTS_JOB_ID,
WEEKLY_SPEND_REPORT_JOB_ID,
)
@@ -5522,20 +5523,44 @@ def _current_general_settings() -> Mapping[str, object]:
return general_settings
-_CLEANUP_SCHEDULE_KEYS: Final = (
+_RETENTION_RULE_KEYS: Final = (
"maximum_spend_logs_retention_period",
"maximum_autorouter_session_retention_period",
"maximum_health_check_retention_period",
"maximum_daily_tag_spend_retention_period",
+)
+_CLEANUP_SCHEDULE_KEYS: Final = (
+ *_RETENTION_RULE_KEYS,
"maximum_spend_logs_cleanup_cron",
"maximum_spend_logs_retention_interval",
+ "maximum_spend_logs_size",
+ "maximum_spend_logs_cleanup_target_size",
+ "maximum_spend_logs_size_check_interval",
)
+_SPEND_LOG_CLEANUP_JOB_ID: Final = "spend_log_cleanup_job"
+_SPEND_LOG_SIZE_CHECK_JOB_ID: Final = "spend_log_size_check_job"
def _cleanup_schedule_of(settings: Mapping[str, object]) -> tuple[object, ...]:
return tuple(settings.get(key) for key in _CLEANUP_SCHEDULE_KEYS)
+def _wants_retention_job(settings: Mapping[str, object]) -> bool:
+ """The retention job runs the age rules, and the size rule too when one cron drives both."""
+ return any(settings.get(key) is not None for key in _RETENTION_RULE_KEYS) or (
+ settings.get("maximum_spend_logs_size") is not None and bool(settings.get("maximum_spend_logs_cleanup_cron"))
+ )
+
+
+def _wants_size_check_job(settings: Mapping[str, object]) -> bool:
+ """The size rule gets its own job on maximum_spend_logs_size_check_interval unless a cleanup cron is set."""
+ return settings.get("maximum_spend_logs_size") is not None and not settings.get("maximum_spend_logs_cleanup_cron")
+
+
+def _wants_cleanup_jobs(settings: Mapping[str, object]) -> bool:
+ return _wants_retention_job(settings) or _wants_size_check_job(settings)
+
+
@lru_cache(maxsize=4096)
def _log_ignored_cost_map_copy(model_id: str, fields: tuple[str, ...]) -> None:
verbose_proxy_logger.warning(
@@ -8052,39 +8077,73 @@ class ProxyConfig:
if scheduler is None:
return
- wants_job: Final = any(
- general_settings.get(key) is not None
- for key in (
- "maximum_spend_logs_retention_period",
- "maximum_autorouter_session_retention_period",
- "maximum_health_check_retention_period",
- "maximum_daily_tag_spend_retention_period",
- )
- )
- if not wants_job:
- if scheduler.get_job("spend_log_cleanup_job") is not None:
- scheduler.remove_job("spend_log_cleanup_job")
- verbose_proxy_logger.info("Removed existing spend log cleanup job")
- return
-
- trigger: Final = self._spend_log_cleanup_trigger()
- if trigger is None:
- return
from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import (
SpendLogCleanup,
)
- scheduler.add_job(
- SpendLogCleanup().cleanup_old_spend_logs,
- trigger,
+ if not _wants_retention_job(general_settings):
+ if scheduler.get_job(_SPEND_LOG_CLEANUP_JOB_ID) is not None:
+ scheduler.remove_job(_SPEND_LOG_CLEANUP_JOB_ID)
+ verbose_proxy_logger.info("Removed existing spend log cleanup job")
+ else:
+ trigger: Final = self._spend_log_cleanup_trigger()
+ if trigger is not None:
+ scheduler.add_job(
+ SpendLogCleanup().cleanup_old_spend_logs,
+ trigger,
+ args=[prisma_client],
+ id=_SPEND_LOG_CLEANUP_JOB_ID,
+ replace_existing=True,
+ misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
+ )
+ verbose_proxy_logger.info("Spend log cleanup rescheduled with trigger: %s", trigger)
+
+ if not _wants_size_check_job(general_settings): # pyright: ignore[reportUnknownArgumentType] # general_settings is an untyped module dict
+ if scheduler.get_job(_SPEND_LOG_SIZE_CHECK_JOB_ID) is not None: # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped
+ scheduler.remove_job(_SPEND_LOG_SIZE_CHECK_JOB_ID) # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped
+ verbose_proxy_logger.info("Removed existing spend log size check job")
+ return
+ size_trigger: Final = self._spend_log_size_check_trigger()
+ if size_trigger is None:
+ return
+ scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped
+ SpendLogCleanup().cleanup_spend_logs_by_size,
+ size_trigger,
args=[prisma_client],
- id="spend_log_cleanup_job",
+ id=_SPEND_LOG_SIZE_CHECK_JOB_ID,
replace_existing=True,
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
)
- verbose_proxy_logger.info("Spend log cleanup rescheduled with trigger: %s", trigger)
+ verbose_proxy_logger.info("Spend log size check rescheduled with trigger: %s", size_trigger)
- def _spend_log_cleanup_trigger(self) -> BaseTrigger | None:
+ def _spend_log_size_check_trigger(self) -> object | None:
+ return self._staggered_interval_trigger(
+ _SPEND_LOG_SIZE_CHECK_JOB_ID,
+ "maximum_spend_logs_size_check_interval",
+ general_settings.get("maximum_spend_logs_size_check_interval", SPEND_LOG_SIZE_CHECK_INTERVAL), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # general_settings is an untyped module dict
+ )
+
+ @staticmethod
+ def _staggered_interval_trigger(job_id: str, setting_name: str, interval: object) -> object | None:
+ if not isinstance(interval, str):
+ verbose_proxy_logger.error("Invalid %s value: %r", setting_name, interval)
+ return None
+ # this runs against a started scheduler, which the startup stagger sweep
+ # cannot reach, so the offset is applied here or the job reconverges across
+ # replicas the first time an admin edits the settings
+ try:
+ interval_seconds: Final = duration_in_seconds(interval)
+ return stagger_trigger(
+ job_id=job_id,
+ trigger=IntervalTrigger(seconds=interval_seconds),
+ period_seconds=interval_seconds,
+ settings=parse_stagger_settings(general_settings), # pyright: ignore[reportUnknownArgumentType] # general_settings is an untyped module dict
+ )
+ except (ValueError, OverflowError):
+ verbose_proxy_logger.error("Invalid %s value: %r", setting_name, interval)
+ return None
+
+ def _spend_log_cleanup_trigger(self) -> object | None:
cleanup_cron: Final[object] = general_settings.get("maximum_spend_logs_cleanup_cron")
if cleanup_cron:
from apscheduler.triggers.cron import CronTrigger
@@ -8095,24 +8154,11 @@ class ProxyConfig:
verbose_proxy_logger.error("Invalid maximum_spend_logs_cleanup_cron value: %s", cleanup_cron)
return None
return cron_trigger
- retention_interval: Final[object] = general_settings.get("maximum_spend_logs_retention_interval", "1d")
- if not isinstance(retention_interval, str):
- verbose_proxy_logger.error("Invalid maximum_spend_logs_retention_interval value: %r", retention_interval)
- return None
- # this runs against a started scheduler, which the startup stagger sweep
- # cannot reach, so the offset is applied here or the job reconverges across
- # replicas the first time an admin edits the retention settings
- try:
- interval_seconds: Final = duration_in_seconds(retention_interval)
- return stagger_trigger(
- job_id="spend_log_cleanup_job",
- trigger=IntervalTrigger(seconds=interval_seconds),
- period_seconds=interval_seconds,
- settings=parse_stagger_settings(general_settings),
- )
- except (ValueError, OverflowError):
- verbose_proxy_logger.error("Invalid maximum_spend_logs_retention_interval value: %r", retention_interval)
- return None
+ return self._staggered_interval_trigger(
+ _SPEND_LOG_CLEANUP_JOB_ID,
+ "maximum_spend_logs_retention_interval",
+ general_settings.get("maximum_spend_logs_retention_interval", "1d"),
+ )
async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | None) -> None:
global general_settings
@@ -8269,8 +8315,11 @@ class ProxyConfig:
if scheduler is not None and scheduler.state == STATE_STOPPED:
return
schedule: Final = self._resolved_cleanup_schedule()
- wants_job: Final = any(value is not None for value in schedule[:4])
- has_job: Final = scheduler is not None and scheduler.get_job("spend_log_cleanup_job") is not None
+ wants_job: Final = _wants_cleanup_jobs(self.settings)
+ has_job: Final = scheduler is not None and (
+ scheduler.get_job(_SPEND_LOG_CLEANUP_JOB_ID) is not None
+ or scheduler.get_job(_SPEND_LOG_SIZE_CHECK_JOB_ID) is not None # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped
+ )
baseline: Final = (
self._last_cleanup_schedule_attempt
if has_job and self._last_cleanup_schedule_attempt is not None
@@ -11417,12 +11466,29 @@ class ProxyStartupEvent:
### SPEND LOG CLEANUP ###
cleanup_settings: Final = _current_general_settings()
- if (
- cleanup_settings.get("maximum_spend_logs_retention_period") is not None
- or cleanup_settings.get("maximum_autorouter_session_retention_period") is not None
- or cleanup_settings.get("maximum_health_check_retention_period") is not None
- or cleanup_settings.get("maximum_daily_tag_spend_retention_period") is not None
- ):
+ if _wants_size_check_job(cleanup_settings):
+ size_check_interval: Final = cleanup_settings.get(
+ "maximum_spend_logs_size_check_interval", SPEND_LOG_SIZE_CHECK_INTERVAL
+ )
+ try:
+ if not isinstance(size_check_interval, str):
+ raise TypeError(size_check_interval)
+ size_check_seconds: Final = duration_in_seconds(size_check_interval)
+ scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped
+ SpendLogCleanup().cleanup_spend_logs_by_size,
+ "interval",
+ seconds=size_check_seconds + random.randint(0, 60),
+ args=[prisma_client],
+ id=_SPEND_LOG_SIZE_CHECK_JOB_ID,
+ replace_existing=True,
+ misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
+ )
+ verbose_proxy_logger.info("Spend log size check scheduled every %s", size_check_interval)
+ except (TypeError, ValueError, OverflowError):
+ verbose_proxy_logger.error(
+ "Invalid maximum_spend_logs_size_check_interval value: %r", size_check_interval
+ )
+ if _wants_retention_job(cleanup_settings):
spend_log_cleanup: Final = SpendLogCleanup()
cleanup_cron: Final = cleanup_settings.get("maximum_spend_logs_cleanup_cron")
@@ -11437,7 +11503,7 @@ class ProxyStartupEvent:
spend_log_cleanup.cleanup_old_spend_logs,
cron_trigger,
args=[prisma_client],
- id="spend_log_cleanup_job",
+ id=_SPEND_LOG_CLEANUP_JOB_ID,
replace_existing=True,
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
)
@@ -11456,7 +11522,7 @@ class ProxyStartupEvent:
"interval",
seconds=interval_seconds + random.randint(0, 60),
args=[prisma_client],
- id="spend_log_cleanup_job",
+ id=_SPEND_LOG_CLEANUP_JOB_ID,
replace_existing=True,
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
)
@@ -11464,6 +11530,7 @@ class ProxyStartupEvent:
verbose_proxy_logger.error(
"Invalid maximum_spend_logs_retention_interval value: %r", retention_interval
)
+ if _wants_cleanup_jobs(cleanup_settings):
proxy_config.record_cleanup_schedule_attempt(cleanup_settings)
### CHECK BATCH COST ###
if llm_router is not None and PROXY_BATCH_POLLING_ENABLED:
diff --git a/tests/integration/database/test_cleanup_transactions.py b/tests/integration/database/test_cleanup_transactions.py
index 925fbaf6f01..72da2bb48ee 100644
--- a/tests/integration/database/test_cleanup_transactions.py
+++ b/tests/integration/database/test_cleanup_transactions.py
@@ -105,3 +105,166 @@ async def test_delete_batch_survives_witnessed_lock_past_transaction_default(cap
await database.disconnect()
finally:
setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
+
+
+async def test_size_rule_deletes_the_oldest_rows_until_under_the_target_and_does_not_refire_after_vacuum(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """
+ Deleting rows never shrinks the table file, so the size pass must stop on live bytes, not on
+ pg_total_relation_size, taking the oldest rows first and keeping the tool index on the same cutoff. Once
+ autovacuum has turned the deleted rows into free space the catalog no longer shows them as dead, and the
+ files still measure the same; the next check must read what the last run freed from its saved count and
+ leave the table alone instead of calling the whole file live again.
+ """
+ caplog.set_level(logging.INFO, logger="LiteLLM Proxy")
+ schema: Final = f"integration_{uuid.uuid4().hex}"
+ url: Final = os.environ["DATABASE_URL"]
+ parsed: Final = urlsplit(url)
+ scoped_url: Final = urlunsplit(
+ parsed._replace(query=urlencode({**dict(parse_qsl(parsed.query)), "schema": schema}))
+ )
+ logs: Final = sql.Identifier(schema, "LiteLLM_SpendLogs")
+ index: Final = sql.Identifier(schema, "LiteLLM_SpendLogToolIndex")
+ config: Final = sql.Identifier(schema, "LiteLLM_Config")
+ seeded: Final = 6000
+ now: Final = datetime.now(timezone.utc).replace(tzinfo=None)
+ stats_sql: Final = (
+ "SELECT sum(n_live_tup)::int AS live, sum(n_dead_tup)::int AS dead, sum(n_tup_ins)::int AS inserted "
+ "FROM pg_stat_user_tables "
+ "WHERE schemaname = %s AND relname IN ('LiteLLM_SpendLogs', 'LiteLLM_SpendLogToolIndex')"
+ )
+ with psycopg.connect(url, autocommit=True) as setup:
+ setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
+ try:
+ setup.execute(
+ sql.SQL(
+ 'CREATE TABLE {} (request_id text, "startTime" timestamp NOT NULL, payload text, '
+ 'PRIMARY KEY (request_id, "startTime"))'
+ ).format(logs)
+ )
+ setup.execute(
+ sql.SQL(
+ "CREATE TABLE {} (request_id text, tool_name text, start_time timestamp NOT NULL, "
+ "PRIMARY KEY (request_id, tool_name))"
+ ).format(index)
+ )
+ setup.execute(sql.SQL("CREATE TABLE {} (param_name text PRIMARY KEY, param_value jsonb)").format(config))
+ setup.execute(
+ sql.SQL(
+ "INSERT INTO {} SELECT 'req-' || lpad(i::text, 6, '0'), %s - (%s - i) * interval '1 minute', "
+ "repeat('x', 1000) FROM generate_series(1, %s) AS i"
+ ).format(logs),
+ (now, seeded, seeded),
+ )
+ setup.execute(
+ sql.SQL(
+ "INSERT INTO {} SELECT request_id, 'tool', \"startTime\" FROM {} WHERE right(request_id, 1) = '0'"
+ ).format(index, logs)
+ )
+ index_rows: Final = seeded // 10
+ inserts_deadline: Final = time.monotonic() + 10
+ while True:
+ # a backend only ships its pending row counters when it runs another command, so give the
+ # seeding connection one; the size rule rolls its count forward from n_tup_ins, which must
+ # show the seed inserts before ANALYZE sets the live estimate or the two add up to double
+ setup.execute("SELECT 1")
+ (inserts,) = read_rows(stats_sql, (schema,))
+ if inserts["inserted"] == seeded + index_rows:
+ break
+ assert time.monotonic() < inserts_deadline, f"row statistics never saw the seed inserts: {inserts}"
+ await asyncio.sleep(0.1)
+ setup.execute(sql.SQL("ANALYZE {}").format(logs))
+ setup.execute(sql.SQL("ANALYZE {}").format(index))
+ stats_deadline: Final = time.monotonic() + 10
+ while True:
+ setup.execute("SELECT 1")
+ (stats,) = read_rows(stats_sql, (schema,))
+ if stats["live"] == seeded + index_rows:
+ break
+ assert time.monotonic() < stats_deadline, f"row statistics never caught up with ANALYZE: {stats}"
+ await asyncio.sleep(0.1)
+ (allocated,) = read_rows(
+ "SELECT (pg_total_relation_size(%s::regclass) + pg_total_relation_size(%s::regclass))::bigint "
+ "AS allocated",
+ (logs.as_string(setup), index.as_string(setup)),
+ )
+ allocated_before: Final = int(str(allocated["allocated"]))
+ assert allocated_before > 0
+
+ cleanup: Final = SpendLogCleanup(
+ general_settings={
+ # trigger below what is there, clean down to half of it
+ "maximum_spend_logs_size": str(allocated_before * 3 // 4),
+ "maximum_spend_logs_cleanup_target_size": str(allocated_before // 2),
+ "maximum_spend_logs_cleanup_batch_size": 100,
+ }
+ )
+ cleanup.pod_lock_manager = None
+
+ async def size_check() -> None:
+ database: Final = Prisma(datasource={"url": scoped_url})
+ await database.connect()
+ try:
+ await cleanup.cleanup_spend_logs_by_size(PartitionConnection(database))
+ finally:
+ await database.disconnect()
+
+ await size_check()
+
+ remaining: Final = read_rows(
+ f'SELECT request_id FROM {logs.as_string(setup)} ORDER BY "startTime"',
+ (),
+ )
+ # half the bytes is half the rows: the oldest 3000 spend logs and their 300 index rows go, counted
+ # from the deletes themselves, so the stats collector's lag cannot make the run overshoot. A page the
+ # files grew by between this measurement and the run's probe costs at most one more batch
+ assert 2900 <= len(remaining) <= 3000, f"{len(remaining)} spend logs remain, expected 3000 less one batch"
+ assert remaining[0]["request_id"] == f"req-{seeded - len(remaining) + 1:06d}"
+ assert remaining[-1]["request_id"] == f"req-{seeded:06d}"
+ (orphans,) = read_rows(
+ f"SELECT count(*)::int AS n FROM {index.as_string(setup)} i "
+ f"WHERE NOT EXISTS (SELECT 1 FROM {logs.as_string(setup)} l WHERE l.request_id = i.request_id)",
+ (),
+ )
+ assert orphans["n"] == 0
+ (kept_index,) = read_rows(f"SELECT count(*)::int AS n FROM {index.as_string(setup)}", ())
+ assert kept_index["n"] == len(remaining) // 10
+ summary: Final = next(
+ record.getMessage()
+ for record in caplog.records
+ if "Spend log cleanup run finished" in record.getMessage()
+ )
+ assert "target_reached=true" in summary
+ assert f"allocated_before={allocated_before}" in summary
+ # the file did not shrink: that is the whole reason the reclaimable count exists
+ assert f"allocated_after={allocated_before}" in summary
+ assert "VACUUM FULL or pg_repack" in summary
+
+ deletes_deadline: Final = time.monotonic() + 10
+ while True:
+ (after_deletes,) = read_rows(stats_sql, (schema,))
+ if after_deletes["dead"] == 3300:
+ break
+ assert time.monotonic() < deletes_deadline, f"row statistics never saw the deletes: {after_deletes}"
+ await asyncio.sleep(0.1)
+ setup.execute(sql.SQL("VACUUM (ANALYZE) {}").format(logs))
+ setup.execute(sql.SQL("VACUUM (ANALYZE) {}").format(index))
+ vacuum_deadline: Final = time.monotonic() + 10
+ while True:
+ (after_vacuum,) = read_rows(stats_sql, (schema,))
+ if after_vacuum["dead"] == 0:
+ break
+ assert time.monotonic() < vacuum_deadline, f"row statistics never caught up with VACUUM: {after_vacuum}"
+ await asyncio.sleep(0.1)
+ caplog.clear()
+
+ await size_check()
+
+ (still_there,) = read_rows(f"SELECT count(*)::int AS n FROM {logs.as_string(setup)}", ())
+ assert still_there["n"] == len(remaining), (
+ "the second check deleted rows the first one had already brought under the limit"
+ )
+ assert any("is under maximum_spend_logs_size" in record.getMessage() for record in caplog.records)
+ finally:
+ setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
diff --git a/tests/unit/proxy/test_spend_log_cleanup.py b/tests/unit/proxy/test_spend_log_cleanup.py
index a9bc82d141d..e70b93f7955 100644
--- a/tests/unit/proxy/test_spend_log_cleanup.py
+++ b/tests/unit/proxy/test_spend_log_cleanup.py
@@ -1681,7 +1681,7 @@ async def test_a_cleanup_delete_batch_renders_a_postgres_delete_span_for_its_tab
assert await postgres_span_names() == ("postgres.delete LiteLLM_SpendLogs",)
-# --- spend log storage measurement and the reclaimable-bytes state ---------------------------------------
+# --- size rule (maximum_spend_logs_size) ---------------------------------------------------------------
def _storage(allocated: int, live: int, dead: int, inserted: int | None = None, deleted: int | None = None) -> dict:
@@ -1694,6 +1694,50 @@ def _storage(allocated: int, live: int, dead: int, inserted: int | None = None,
}
+def _wire_size_probes(
+ db, storage_rows: list[dict], cutoff: datetime, saved_state: SizeCleanupState | None = None
+) -> list[SizeCleanupState]:
+ """
+ Answer the size pass's catalog reads: the storage probe walks `storage_rows` one measurement per call
+ (repeating the last), the step-cutoff query returns `cutoff`, the remaining-rows probe returns 0, and the
+ LiteLLM_Config row holds `saved_state`. Writes of that row land in the returned list instead of
+ db.execute_raw, so DELETE counts stay what tests configure.
+ """
+ measurements = iter(storage_rows)
+ last = [storage_rows[-1]]
+ saves: list[SizeCleanupState] = []
+ deletes = db.execute_raw
+
+ async def _query_raw(sql, *args):
+ if "pg_total_relation_size" in sql:
+ row = next(measurements, None)
+ if row is None:
+ row = last[0]
+ return [row]
+ if "OFFSET" in sql:
+ return [{"startTime": cutoff}]
+ if '"LiteLLM_Config"' in sql:
+ return [{"state_json": saved_state.model_dump_json()}] if saved_state is not None else []
+ return [{"remaining": 0}]
+
+ async def _execute_raw(sql, *args):
+ if '"LiteLLM_Config"' in sql:
+ saves.append(SizeCleanupState.model_validate_json(args[1]))
+ return 1
+ return await deletes(sql, *args)
+
+ db.query_raw = AsyncMock(side_effect=_query_raw)
+ db.execute_raw = _execute_raw
+ db.deletes = deletes
+ return saves
+
+
+def _size_cleaner(settings: dict, **kwargs) -> SpendLogCleanup:
+ cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_size": "1GB", **settings}, **kwargs)
+ cleaner.pod_lock_manager = None
+ return cleaner
+
+
@pytest.mark.parametrize(
("value", "expected"),
[
@@ -1719,6 +1763,160 @@ def test_size_in_bytes_rejects_what_cannot_be_a_size(value):
size_in_bytes(value)
+def test_an_unset_size_means_no_size_rule_and_a_bad_target_falls_back_to_the_size(caplog):
+ assert SpendLogCleanup(general_settings={})._size_rule() is None
+ rule = _size_cleaner({"maximum_spend_logs_cleanup_target_size": "200MB"})._size_rule()
+ assert rule is not None
+ assert (rule.limit_bytes, rule.target_bytes) == (1024**3, 200 * 1024**2)
+ with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
+ rule_without_room = _size_cleaner({"maximum_spend_logs_cleanup_target_size": "2GB"})._size_rule()
+ assert rule_without_room is not None
+ assert rule_without_room.target_bytes == rule_without_room.limit_bytes == 1024**3
+ assert any("must be below maximum_spend_logs_size" in record.getMessage() for record in caplog.records)
+
+
+@pytest.mark.asyncio
+async def test_size_pass_under_the_limit_issues_no_delete():
+ db = MagicMock()
+ _wire_tx(db)
+ db.execute_raw = AsyncMock(return_value=0)
+ _wire_size_probes(db, [_storage(allocated=500 * 1024**2, live=1000, dead=0)], cutoff=datetime.now(timezone.utc))
+ client = MagicMock()
+ client.db = db
+
+ await _size_cleaner({}).cleanup_spend_logs_by_size(client)
+
+ assert db.deletes.await_count == 0
+
+
+@pytest.mark.asyncio
+async def test_size_pass_deletes_oldest_first_in_steps_until_the_live_estimate_is_under_the_target(caplog):
+ """
+ Over the limit, the pass turns the next step of oldest rows into a cutoff and runs the retention DELETE
+ for the spend logs and their tool-index rows, re-measuring after each step; it stops on the first
+ measurement under the target, not when the file size (which deletes never shrink) moves.
+ """
+ db = MagicMock()
+ _wire_tx(db)
+ # 3 GiB over 3000 rows is 1 MiB a row, so 2560 rows take it from 3 GiB to the 512 MiB target: one step
+ # of 3 batches; the tool index is cleaned on the same cutoff
+ db.execute_raw = AsyncMock(side_effect=[1000, 1000, 1000, 300, 0])
+ step_cutoff = datetime(2026, 9, 1, tzinfo=timezone.utc)
+ allocated = 3 * 1024**3
+ _wire_size_probes(db, [_storage(allocated, live=3000, dead=0)], cutoff=step_cutoff)
+ client = MagicMock()
+ client.db = db
+ cleaner = _size_cleaner({"maximum_spend_logs_cleanup_target_size": "512MB"})
+
+ with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"):
+ await cleaner.cleanup_spend_logs_by_size(client)
+
+ deletes = [(call[0][0], call[0][1:]) for call in db.deletes.call_args_list]
+ assert len(deletes) == 5
+ assert 'DELETE FROM "LiteLLM_SpendLogs"' in deletes[0][0]
+ assert 'WHERE "startTime" < $1' in deletes[0][0]
+ assert deletes[0][1][0] == step_cutoff
+ assert 'DELETE FROM "LiteLLM_SpendLogToolIndex"' in deletes[3][0]
+ assert deletes[3][1][0] == step_cutoff
+ offset_query = next(call for call in db.query_raw.call_args_list if "OFFSET" in call[0][0])
+ assert offset_query[0][1] == 3000
+ summary = next(r.getMessage() for r in caplog.records if "Spend log cleanup run finished" in r.getMessage())
+ assert "outcome=completed" in summary
+ assert "LiteLLM_SpendLogs (size rule): deleted=3000 stop_reason=target_reached" in summary
+ assert "LiteLLM_SpendLogToolIndex (size rule): deleted=300 stop_reason=target_reached" in summary
+ assert f"allocated_before={allocated}" in summary
+ assert "estimated_live_before=3221225472" in summary
+ assert "estimated_live_after=0" in summary
+ assert "target_reached=true above_target=0" in summary
+ assert "VACUUM FULL or pg_repack" in summary
+
+
+@pytest.mark.asyncio
+async def test_size_pass_stops_at_the_run_batch_cap_and_reports_how_far_above_the_target_it_is(caplog):
+ db = MagicMock()
+ _wire_tx(db)
+ # 4 GiB over 4000 rows: 3584 rows must go to reach the 512 MiB target, the cap allows 3 batches
+ db.execute_raw = AsyncMock(side_effect=[1000, 1000, 1000, 0])
+ allocated = 4 * 1024**3
+ saves = _wire_size_probes(db, [_storage(allocated, live=4000, dead=0)], cutoff=datetime.now(timezone.utc))
+ client = MagicMock()
+ client.db = db
+ cleaner = _size_cleaner(
+ {"maximum_spend_logs_cleanup_target_size": "512MB", "maximum_spend_logs_cleanup_max_batches": 3}
+ )
+
+ with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+ await cleaner.cleanup_spend_logs_by_size(client)
+
+ assert db.deletes.await_count == 4
+ summary = next(r for r in caplog.records if "Spend log cleanup run finished" in r.getMessage())
+ assert summary.levelno == logging.WARNING
+ assert "outcome=batch_cap_reached" in summary.getMessage()
+ assert "LiteLLM_SpendLogs (size rule): deleted=3000 stop_reason=batch_cap_reached" in summary.getMessage()
+ # 1000 of 4000 rows left is 1 GiB live, 512 MiB above the target
+ assert f"target_reached=false above_target={512 * 1024**2}" in summary.getMessage()
+ # the next check starts from what this run freed, with the deletes already counted out of n_tup_del
+ assert saves == [
+ SizeCleanupState(
+ allocated_bytes=allocated,
+ reclaimable_bytes=round(3000 * allocated / 4000),
+ bytes_per_row=allocated / 4000,
+ inserted_total=4000,
+ deleted_total=3000,
+ )
+ ]
+
+
+@pytest.mark.asyncio
+async def test_size_pass_skips_on_stale_row_statistics_instead_of_trusting_an_estimate_of_zero(caplog):
+ db = MagicMock()
+ _wire_tx(db)
+ db.execute_raw = AsyncMock(return_value=1000)
+ _wire_size_probes(db, [_storage(allocated=5 * 1024**3, live=0, dead=0)], cutoff=datetime.now(timezone.utc))
+ client = MagicMock()
+ client.db = db
+
+ with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+ await _size_cleaner({}).cleanup_spend_logs_by_size(client)
+
+ assert db.deletes.await_count == 0
+ assert any("row statistics are stale" in record.getMessage() for record in caplog.records)
+
+
+@pytest.mark.asyncio
+async def test_size_pass_does_not_refire_after_autovacuum_forgets_the_rows_it_deleted():
+ """
+ The run before this one deleted 15,000 of 20,000 rows and saved what it freed. Autovacuum has since
+ turned those dead tuples into free space, so pg_stat_user_tables reports 5,000 live and 0 dead rows while
+ the files still measure the full 32 MB. Trusting the live share would call all 32 MB live and empty the
+ table; the saved count says 8 MB are live, under the limit, so nothing is deleted.
+ """
+ db = MagicMock()
+ _wire_tx(db)
+ db.execute_raw = AsyncMock(return_value=1000)
+ after_last_run = SizeCleanupState(
+ allocated_bytes=32_030_720,
+ reclaimable_bytes=24_015_000,
+ bytes_per_row=1601.0,
+ inserted_total=20_000,
+ deleted_total=15_000,
+ )
+ saves = _wire_size_probes(
+ db,
+ [_storage(32_030_720, live=5000, dead=0, inserted=20_000, deleted=15_000)],
+ cutoff=datetime.now(timezone.utc),
+ saved_state=after_last_run,
+ )
+ client = MagicMock()
+ client.db = db
+ cleaner = _size_cleaner({"maximum_spend_logs_size": "16MB", "maximum_spend_logs_cleanup_target_size": "10MB"})
+
+ await cleaner.cleanup_spend_logs_by_size(client)
+
+ assert db.deletes.await_count == 0
+ assert saves[0].reclaimable_bytes == 24_015_000
+
+
_RUN_STATE = SizeCleanupState(
allocated_bytes=32_030_720,
reclaimable_bytes=24_015_000,
@@ -1765,6 +1963,58 @@ def test_advance_size_state_tracks_free_space_through_vacuum_inserts_and_other_d
assert 0 <= probe["allocated_bytes"] - advanced.reclaimable_bytes <= probe["allocated_bytes"], name
+@pytest.mark.asyncio
+async def test_size_only_run_leaves_the_age_rules_to_the_retention_job():
+ db = MagicMock()
+ _wire_tx(db)
+ db.execute_raw = AsyncMock(return_value=0)
+ _wire_size_probes(db, [_storage(allocated=10 * 1024**3, live=100, dead=0)], cutoff=datetime.now(timezone.utc))
+ client = MagicMock()
+ client.db = db
+ cleaner = _size_cleaner(
+ {"maximum_spend_logs_retention_period": "7d", "maximum_health_check_retention_period": "30d"}
+ )
+
+ await cleaner.cleanup_spend_logs_by_size(client)
+
+ tables = {call[0][0].split('"')[1] for call in db.deletes.call_args_list}
+ assert tables == {"LiteLLM_SpendLogs", "LiteLLM_SpendLogToolIndex"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("cron", "size_pass_expected"), [(None, False), ("0 3 * * *", True)])
+async def test_the_retention_run_carries_the_size_pass_only_when_one_cron_drives_both(cron, size_pass_expected):
+ db = MagicMock()
+ _wire_tx(db)
+ db.execute_raw = AsyncMock(return_value=0)
+ _wire_size_probes(db, [_storage(allocated=10 * 1024**3, live=100, dead=0)], cutoff=datetime.now(timezone.utc))
+ client = MagicMock()
+ client.db = db
+ cleaner = _size_cleaner({"maximum_spend_logs_retention_period": "7d", "maximum_spend_logs_cleanup_cron": cron})
+
+ await cleaner.cleanup_old_spend_logs(client)
+
+ measured = any("pg_total_relation_size" in call[0][0] for call in db.query_raw.call_args_list)
+ assert measured is size_pass_expected
+
+
+@pytest.mark.parametrize(
+ ("settings", "retention_job", "size_job"),
+ [
+ ({}, False, False),
+ ({"maximum_spend_logs_retention_period": "7d"}, True, False),
+ ({"maximum_spend_logs_size": "180GB"}, False, True),
+ ({"maximum_spend_logs_size": "180GB", "maximum_spend_logs_retention_period": "7d"}, True, True),
+ ({"maximum_spend_logs_size": "180GB", "maximum_spend_logs_cleanup_cron": "0 3 * * *"}, True, False),
+ ],
+)
+def test_which_scheduler_jobs_the_cleanup_settings_ask_for(settings, retention_job, size_job):
+ from litellm.proxy.proxy_server import _wants_retention_job, _wants_size_check_job
+
+ assert _wants_retention_job(settings) is retention_job
+ assert _wants_size_check_job(settings) is size_job
+
+
def test_after_deleting_counts_the_rows_and_any_file_growth_as_reclaimable_and_to_state_keeps_the_counters():
storage = SpendLogStorage(allocated_bytes=32_030_720, reclaimable_bytes=0, bytes_per_row=1601.0, live_rows=20_000)
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 2b6898e39d9..b93f7dce2ca 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -30072,11 +30072,26 @@ export interface components {
* @description Wall-clock budget for one spend log cleanup run (e.g. '5m'), shared across every table it prunes. A run that hits the budget stops and the next run resumes from where it left off. Defaults to '5m'.
*/
maximum_spend_logs_cleanup_run_budget?: string | null;
+ /**
+ * Maximum Spend Logs Cleanup Target Size
+ * @description Level a size-triggered cleanup run deletes down to (e.g. '20GB'). Must be below maximum_spend_logs_size. Unset means the run stops as soon as the tables are back under maximum_spend_logs_size.
+ */
+ maximum_spend_logs_cleanup_target_size?: string | null;
/**
* Maximum Spend Logs Retention Period
* @description Maximum retention period for spend logs (e.g., '7d' for 7 days). Logs older than this will be deleted.
*/
maximum_spend_logs_retention_period?: string | null;
+ /**
+ * Maximum Spend Logs Size
+ * @description Size rule for spend logs (e.g. '180GB'): when LiteLLM_SpendLogs plus LiteLLM_SpendLogToolIndex (heap, indexes and TOAST, measured with pg_total_relation_size) hold more live data than this, the cleanup job deletes the oldest spend logs first until they are under maximum_spend_logs_cleanup_target_size. Independent of maximum_spend_logs_retention_period: a row is deleted when either rule says so. Unset means no size rule.
+ */
+ maximum_spend_logs_size?: string | null;
+ /**
+ * Maximum Spend Logs Size Check Interval
+ * @description How often the size rule is checked (e.g. '10m'). Defaults to '1d'. Ignored when maximum_spend_logs_cleanup_cron is set, which then drives both the age and the size cleanup.
+ */
+ maximum_spend_logs_size_check_interval?: string | null;
/**
* Mcp Advertised Versions
* @description MCP revisions enabled by the gateway. Defaults to all completed legacy revisions. Modern protocol serving requires explicit opt-in. Apps/Tasks remain disabled.
From e63aec029a9111616d039712aec0df498bbfbd1b Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 10:50:26 -0700
Subject: [PATCH 011/203] feat(proxy): opt-in free-tier allowance for Google
Search grounding spend (#47494)
* feat(proxy): opt-in free-tier allowance for Google Search grounding spend
* fix(proxy): declare the search free tier counter's service target
* test(cost_map): allow search_free_requests in the cost map schema
* fix(proxy): skip cache hits and never free more than the billed cost in the search free tier
---------
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
litellm/__init__.py | 1 +
litellm/constants.py | 1 +
...odel_prices_and_context_window_backup.json | 47 +++++
.../proxy/hooks/proxy_track_cost_callback.py | 18 +-
litellm/proxy/hooks/search_free_tier.py | 123 +++++++++++
litellm/types/utils.py | 7 +
litellm/utils.py | 1 +
model_prices_and_context_window.json | 47 +++++
.../hooks/test_proxy_track_cost_callback.py | 90 +++++++-
.../unit/proxy/hooks/test_search_free_tier.py | 194 ++++++++++++++++++
.../test_model_prices_and_context_window.py | 28 +++
tests/unit/test_utils.py | 10 +
12 files changed, 564 insertions(+), 3 deletions(-)
create mode 100644 litellm/proxy/hooks/search_free_tier.py
create mode 100644 tests/unit/proxy/hooks/test_search_free_tier.py
diff --git a/litellm/__init__.py b/litellm/__init__.py
index 0fe3d091c2a..f6604a3b4e5 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -551,6 +551,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
+apply_search_free_tier: bool = False
cost_margin_config: Dict[
str, Union[float, Dict[str, float]]
] = {} # Provider-specific or global cost margins. Examples:
diff --git a/litellm/constants.py b/litellm/constants.py
index 377baa25000..a0dcf44d121 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1979,6 +1979,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"public_model_groups",
"public_model_groups_links",
"cost_discount_config",
+ "apply_search_free_tier",
"cost_margin_config",
"block_requests_for_models_without_pricing",
"budget_exceeded_throttle_percentage",
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 903210c57e5..503877a9d4d 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -28948,6 +28948,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-07,
"input_cost_per_token_batches": 1.5e-07,
@@ -29062,6 +29063,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3-pro-image-preview": {
@@ -29114,6 +29116,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image": {
@@ -29164,6 +29167,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image-preview": {
@@ -29207,6 +29211,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-lite-image": {
@@ -29302,6 +29307,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -29363,6 +29369,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@@ -29422,6 +29429,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -29508,6 +29516,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-08,
"input_cost_per_token_batches": 5e-08,
@@ -29562,6 +29571,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"supports_image_size": false
},
@@ -29608,6 +29618,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"supports_image_size": false
},
@@ -29795,6 +29806,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_token_above_200k_tokens_priority": 4.5e-06,
"input_cost_per_token_batches": 6.25e-07,
@@ -29832,6 +29844,7 @@
"output_cost_per_token_priority": 2.16e-05,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -29912,6 +29925,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -29968,6 +29982,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3-flash-preview": {
@@ -29992,6 +30007,7 @@
"output_cost_per_token_flex": 1.5e-06,
"output_cost_per_token_priority": 5.4e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -30074,6 +30090,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@@ -30138,6 +30155,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -30198,6 +30216,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -30224,6 +30243,7 @@
"prompt_cache_min_tokens": 4096.0,
"regional_endpoint_uplift_multiplier": 1.1,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -30369,6 +30389,7 @@
"output_cost_per_token_priority": 2.16e-05,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -30455,6 +30476,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -30782,6 +30804,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"supports_reasoning": false
},
@@ -30826,6 +30849,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.1-flash-lite-preview": {
@@ -30887,6 +30911,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@@ -31036,6 +31061,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-07,
"input_cost_per_audio_token_priority": 1.8e-06,
@@ -31148,6 +31174,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"supports_reasoning": true
},
@@ -31212,6 +31239,7 @@
"output_cost_per_token_batches": 1.5e-06,
"rpm": 1000.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -31362,6 +31390,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 1.5e-07,
"input_cost_per_audio_token_priority": 5.4e-07,
@@ -31567,6 +31596,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_token_batches": 6.25e-07,
"input_cost_per_token_flex": 6.25e-07,
@@ -31664,6 +31694,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07,
@@ -31725,6 +31756,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -31781,6 +31813,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 5e-07,
@@ -31844,6 +31877,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@@ -31909,6 +31943,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -31970,6 +32005,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -32031,6 +32067,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -32126,6 +32163,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
@@ -32189,6 +32227,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
@@ -32217,6 +32256,7 @@
"output_cost_per_token_flex": 1.5e-06,
"output_cost_per_token_priority": 5.4e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -32331,6 +32371,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@@ -32395,6 +32436,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -32455,6 +32497,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -32480,6 +32523,7 @@
"output_cost_per_token_priority": 6.75e-06,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -52449,6 +52493,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -52511,6 +52556,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@@ -52571,6 +52617,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py
index 1bfb04ab883..13a205d7e9f 100644
--- a/litellm/proxy/hooks/proxy_track_cost_callback.py
+++ b/litellm/proxy/hooks/proxy_track_cost_callback.py
@@ -1,7 +1,7 @@
import asyncio
import traceback
from collections.abc import Callable, Mapping, Sequence
-from datetime import datetime
+from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol, cast
@@ -33,6 +33,7 @@ from litellm.proxy.db.db_spend_update_writer import (
debitable_model_access_groups,
get_llm_router,
)
+from litellm.proxy.hooks.search_free_tier import apply_search_free_tier
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_route
from litellm.proxy.spend_tracking.spend_counter_batch import post_call_counter_keys, spend_counter_batch_scope
from litellm.proxy.spend_tracking.spend_event import (
@@ -487,8 +488,21 @@ class _ProxyDBLogger(CustomLogger):
key_alias: Final = cast(str | None, metadata.get("user_api_key_alias", None))
end_user_max_budget: Final = metadata.get("user_api_end_user_max_budget", None)
sl_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
+ raw_cost: Final = _str_keyed_fields(
+ sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
+ ).get("response_cost")
+ priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None
response_cost: Final = (
- sl_object.get("response_cost", None) if sl_object is not None else kwargs.get("response_cost", None)
+ await apply_search_free_tier(
+ kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
+ sl_object=sl_object, # pyright: ignore[reportUnknownArgumentType] # payload carries untyped lists
+ response_cost=priced_cost,
+ completion_response=completion_response,
+ counter=proxy_logging_obj.internal_usage_cache.dual_cache.redis_cache,
+ now=datetime.now(timezone.utc),
+ )
+ if litellm.apply_search_free_tier and sl_object is not None and priced_cost is not None
+ else priced_cost
)
tags: Final = _get_request_tags_for_cost_tracking(
sl_object=sl_object,
diff --git a/litellm/proxy/hooks/search_free_tier.py b/litellm/proxy/hooks/search_free_tier.py
new file mode 100644
index 00000000000..4ef300dfbac
--- /dev/null
+++ b/litellm/proxy/hooks/search_free_tier.py
@@ -0,0 +1,123 @@
+from collections.abc import MutableMapping
+from datetime import datetime
+from types import MappingProxyType
+from typing import Final, Protocol
+
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
+
+from litellm._internal_context import with_service_target
+from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time
+from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage
+from litellm.llms.gemini.cost_calculator import cost_per_web_search_request
+from litellm.types.utils import CostBreakdown, ModelInfo, SearchFreeRequests, StandardLoggingPayload, Usage
+
+_PERIOD_DURATION: Final = MappingProxyType({"day": "1d", "month": "1mo"})
+_PERIOD_STAMP: Final = MappingProxyType({"day": "%Y-%m-%d", "month": "%Y-%m"})
+_TTL_GRACE_SECONDS: Final = 3600
+_COUNTED_KWARG: Final = "search_free_tier_counted"
+_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown)
+
+
+class SharedCounter(Protocol):
+ async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: ...
+
+
+class _UsageCarrier(BaseModel):
+ model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
+
+ usage: Usage | None = None
+
+
+def freed_units(allowance: int, units: int, total_after: int) -> int:
+ """Units of this request that landed under the allowance, given the shared total after they were added."""
+ return max(0, min(units, allowance - (total_after - units)))
+
+
+def search_free_tier_key(allowance: SearchFreeRequests, now: datetime) -> str:
+ return f"litellm:search_free_tier:{allowance['shared_group']}:{now.strftime(_PERIOD_STAMP[allowance['period']])}"
+
+
+def search_free_tier_ttl(allowance: SearchFreeRequests, now: datetime) -> int:
+ reset: Final = get_next_standardized_reset_time(_PERIOD_DURATION[allowance["period"]], now, "UTC")
+ return int((reset - now).total_seconds()) + _TTL_GRACE_SECONDS
+
+
+def _usage_of(completion_response: object) -> Usage | None:
+ try:
+ return _UsageCarrier.model_validate(completion_response, from_attributes=True).usage
+ except ValidationError:
+ return None
+
+
+def _grounding_queries(usage: Usage) -> int:
+ details: Final = usage.prompt_tokens_details
+ from_details: Final = getattr(details, "web_search_requests", None) if details is not None else None
+ return from_details or get_web_search_requests_from_usage(usage) or 0
+
+
+def _billable_units(usage: Usage, model_info: ModelInfo) -> int:
+ queries: Final = _grounding_queries(usage)
+ if queries == 0:
+ return 0
+ return queries if model_info.get("web_search_billing_unit") == "per_query" else 1
+
+
+def _record_freed_cost(
+ kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it
+ sl_object: StandardLoggingPayload,
+ original_cost: float,
+ freed_cost: float,
+) -> float:
+ reduced_cost: Final = original_cost - freed_cost
+ breakdown: Final = sl_object["cost_breakdown"] or CostBreakdown(total_cost=original_cost)
+ sl_object["cost_breakdown"] = _COST_BREAKDOWN.validate_python( # rebind-ok: the spend row reads this payload
+ {
+ **breakdown,
+ "total_cost": reduced_cost,
+ "original_cost": breakdown.get("original_cost", original_cost),
+ "discount_amount": breakdown.get("discount_amount", 0.0) + freed_cost,
+ }
+ )
+ sl_object["response_cost"] = reduced_cost # rebind-ok: the spend row reads this payload
+ kwargs["response_cost"] = reduced_cost # rebind-ok: the spend row reads kwargs["response_cost"]
+ return reduced_cost
+
+
+@with_service_target("search_free_tier")
+async def apply_search_free_tier(
+ kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it
+ sl_object: StandardLoggingPayload,
+ response_cost: float,
+ completion_response: object,
+ counter: SharedCounter | None,
+ now: datetime,
+) -> float:
+ """Zero the grounding queries that landed under the model's shared free allowance, everything else is unchanged."""
+ model_info: Final = sl_object["model_map_information"]["model_map_value"]
+ allowance: Final = model_info.get("search_free_requests") if model_info is not None else None
+ usage: Final = _usage_of(completion_response)
+ if model_info is None or allowance is None or usage is None or sl_object["cache_hit"]:
+ return response_cost
+ units: Final = _billable_units(usage, model_info)
+ if units == 0 or kwargs.get(_COUNTED_KWARG):
+ return response_cost
+ if counter is None:
+ verbose_proxy_logger.warning(
+ "apply_search_free_tier is on but the proxy has no Redis, grounding is billed at full price"
+ )
+ return response_cost
+ key: Final = search_free_tier_key(allowance, now)
+ try:
+ total_after: Final = await counter.async_increment(key, units, ttl=search_free_tier_ttl(allowance, now))
+ except Exception: # noqa: BLE001 # any counter failure means full price, never a lost spend row
+ verbose_proxy_logger.warning(
+ "search free tier counter %s is unavailable, grounding is billed at full price", key
+ )
+ return response_cost
+ kwargs[_COUNTED_KWARG] = True # rebind-ok: the proxy runs this callback twice per request, count it once
+ freed: Final = freed_units(allowance["count"], units, int(total_after))
+ if freed == 0:
+ return response_cost
+ freed_cost: Final = min(response_cost, cost_per_web_search_request(usage, model_info) * freed / units)
+ return _record_freed_cost(kwargs, sl_object, response_cost, freed_cost)
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 2721a0c1cc1..e268cc90ffd 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -245,6 +245,12 @@ class SearchContextCostPerQuery(TypedDict, total=False):
search_context_size_high: float
+class SearchFreeRequests(TypedDict):
+ count: ReadOnly[int]
+ period: ReadOnly[Literal["day", "month"]]
+ shared_group: ReadOnly[str]
+
+
class AgenticLoopParams(TypedDict, total=False):
"""
Parameters passed to agentic loop hooks (e.g., WebSearch interception).
@@ -436,6 +442,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
web_search_billing_unit: (
Literal["per_query", "per_prompt"] | None
) # "per_query" (Gemini 3.x) or "per_prompt" (Gemini 2.x)
+ search_free_requests: ReadOnly[SearchFreeRequests | None]
google_maps_grounding_cost_per_query: ReadOnly[float | None]
citation_cost_per_token: float | None # Cost per citation token for Perplexity
tiered_pricing: list[dict[str, Any]] | None # Tiered pricing structure for models like Dashscope
diff --git a/litellm/utils.py b/litellm/utils.py
index a6108a15bb3..6b16fc398bd 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -6878,6 +6878,7 @@ def get_model_info_helper(
supports_computer_use=_model_info.get("supports_computer_use", None),
search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None),
web_search_billing_unit=_model_info.get("web_search_billing_unit", None),
+ search_free_requests=_model_info.get("search_free_requests", None),
google_maps_grounding_cost_per_query=_model_info.get("google_maps_grounding_cost_per_query", None),
tpm=_model_info.get("tpm", None),
rpm=_model_info.get("rpm", None),
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 903210c57e5..503877a9d4d 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -28948,6 +28948,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-07,
"input_cost_per_token_batches": 1.5e-07,
@@ -29062,6 +29063,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3-pro-image-preview": {
@@ -29114,6 +29116,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image": {
@@ -29164,6 +29167,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image-preview": {
@@ -29207,6 +29211,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-lite-image": {
@@ -29302,6 +29307,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -29363,6 +29369,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@@ -29422,6 +29429,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -29508,6 +29516,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-08,
"input_cost_per_token_batches": 5e-08,
@@ -29562,6 +29571,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"supports_image_size": false
},
@@ -29608,6 +29618,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"supports_image_size": false
},
@@ -29795,6 +29806,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_token_above_200k_tokens_priority": 4.5e-06,
"input_cost_per_token_batches": 6.25e-07,
@@ -29832,6 +29844,7 @@
"output_cost_per_token_priority": 2.16e-05,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -29912,6 +29925,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -29968,6 +29982,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3-flash-preview": {
@@ -29992,6 +30007,7 @@
"output_cost_per_token_flex": 1.5e-06,
"output_cost_per_token_priority": 5.4e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -30074,6 +30090,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@@ -30138,6 +30155,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -30198,6 +30216,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -30224,6 +30243,7 @@
"prompt_cache_min_tokens": 4096.0,
"regional_endpoint_uplift_multiplier": 1.1,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -30369,6 +30389,7 @@
"output_cost_per_token_priority": 2.16e-05,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -30455,6 +30476,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -30782,6 +30804,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"supports_reasoning": false
},
@@ -30826,6 +30849,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.1-flash-lite-preview": {
@@ -30887,6 +30911,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@@ -31036,6 +31061,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-07,
"input_cost_per_audio_token_priority": 1.8e-06,
@@ -31148,6 +31174,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"supports_reasoning": true
},
@@ -31212,6 +31239,7 @@
"output_cost_per_token_batches": 1.5e-06,
"rpm": 1000.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -31362,6 +31390,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 1.5e-07,
"input_cost_per_audio_token_priority": 5.4e-07,
@@ -31567,6 +31596,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
+ "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_token_batches": 6.25e-07,
"input_cost_per_token_flex": 6.25e-07,
@@ -31664,6 +31694,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07,
@@ -31725,6 +31756,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -31781,6 +31813,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 5e-07,
@@ -31844,6 +31877,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@@ -31909,6 +31943,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -31970,6 +32005,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -32031,6 +32067,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -32126,6 +32163,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
@@ -32189,6 +32227,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
@@ -32217,6 +32256,7 @@
"output_cost_per_token_flex": 1.5e-06,
"output_cost_per_token_priority": 5.4e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -32331,6 +32371,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@@ -32395,6 +32436,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -32455,6 +32497,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -32480,6 +32523,7 @@
"output_cost_per_token_priority": 6.75e-06,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@@ -52449,6 +52493,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@@ -52511,6 +52556,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@@ -52571,6 +52617,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
+ "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
index 1132d339f15..6be4b93c173 100644
--- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
+++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
@@ -29,7 +29,7 @@ from litellm.proxy.utils import ProxyUpdateSpend
from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event
from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
-from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, Usage
+from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage
@pytest.mark.asyncio
@@ -3182,3 +3182,91 @@ async def test_track_cost_callback_enqueue_emits_no_service_span(): # test-qual
assert update_database.await_count == 1, "the spend enqueue itself must still run"
assert success_hook.await_count == 0, [call.kwargs for call in success_hook.await_args_list]
+
+class _SharedCounter:
+ def __init__(self) -> None:
+ self.totals: dict[str, float] = {}
+
+ async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
+ self.totals[key] = self.totals.get(key, 0) + value
+ return self.totals[key]
+
+
+def _grounded_gemini_kwargs(response_cost: float) -> dict:
+ return {
+ "model": "vertex_ai/gemini-test",
+ "call_type": "acompletion",
+ "response_cost": response_cost,
+ "litellm_params": {"metadata": {"user_api_key": "sk-test", "user_api_key_user_id": "user-1"}},
+ "standard_logging_object": {
+ "response_cost": response_cost,
+ "request_tags": None,
+ "cache_hit": None,
+ "cost_breakdown": {
+ "input_cost": 0.0,
+ "output_cost": 0.0,
+ "tool_usage_cost": 0.5,
+ "total_cost": response_cost,
+ },
+ "model_map_information": {
+ "model_map_key": "vertex_ai/gemini-test",
+ "model_map_value": {
+ "litellm_provider": "vertex_ai",
+ "mode": "chat",
+ "web_search_billing_unit": "per_query",
+ "search_context_cost_per_query": {"search_context_size_medium": 0.1},
+ "search_free_requests": {"count": 100, "period": "month", "shared_group": "test"},
+ },
+ },
+ },
+ "stream": False,
+ }
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("opted_in", [True, False])
+async def test_track_cost_callback_frees_grounding_under_the_allowance_only_when_opted_in(monkeypatch, opted_in):
+ monkeypatch.setattr(litellm, "apply_search_free_tier", opted_in, raising=False)
+ counter = _SharedCounter()
+ kwargs = _grounded_gemini_kwargs(response_cost=1.0)
+ response = ModelResponse(
+ usage=Usage(
+ prompt_tokens=10,
+ completion_tokens=5,
+ total_tokens=15,
+ prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=5),
+ )
+ )
+
+ with (
+ patch( # test-quality-ok: increment_spend_counters is a proxy_server global the callback reads lazily, no seam
+ "litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock
+ ),
+ patch( # test-quality-ok: update_cache is a proxy_server global the callback reads lazily, no seam
+ "litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock
+ ),
+ patch( # test-quality-ok: callback imports proxy_logging_obj off proxy_server in its body, no seam
+ "litellm.proxy.proxy_server.proxy_logging_obj"
+ ) as mock_proxy_logging,
+ ):
+ mock_proxy_logging.failed_tracking_alert = AsyncMock()
+ mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(return_value=True)
+ mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock()
+ mock_proxy_logging.internal_usage_cache.dual_cache.redis_cache = counter
+
+ await ProxyDBLogger()._PROXY_track_cost_callback(
+ kwargs=kwargs, completion_response=response, start_time=datetime.now(), end_time=datetime.now()
+ )
+ await asyncio.sleep(0)
+
+ mock_proxy_logging.failed_tracking_alert.assert_not_called()
+ mock_proxy_logging.db_spend_update_writer.update_database.assert_awaited_once()
+
+ breakdown = kwargs["standard_logging_object"]["cost_breakdown"]
+ if opted_in:
+ assert kwargs["response_cost"] == pytest.approx(1.0 - 5 * 0.1)
+ assert breakdown["original_cost"] == 1.0 and breakdown["discount_amount"] == pytest.approx(5 * 0.1)
+ assert sum(counter.totals.values()) == 5
+ else:
+ assert kwargs["response_cost"] == 1.0 and "discount_amount" not in breakdown
+ assert counter.totals == {}
diff --git a/tests/unit/proxy/hooks/test_search_free_tier.py b/tests/unit/proxy/hooks/test_search_free_tier.py
new file mode 100644
index 00000000000..3dc89c0dc72
--- /dev/null
+++ b/tests/unit/proxy/hooks/test_search_free_tier.py
@@ -0,0 +1,194 @@
+from datetime import datetime, timedelta, timezone
+from typing import Final
+
+import pytest
+
+from litellm.proxy.hooks.search_free_tier import (
+ apply_search_free_tier,
+ freed_units,
+ search_free_tier_key,
+ search_free_tier_ttl,
+)
+from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
+
+NOW: Final = datetime(2026, 10, 20, 12, 30, tzinfo=timezone.utc)
+RATE: Final = 0.01
+
+
+class _Counter:
+ def __init__(self) -> None:
+ self.totals: dict[str, float] = {}
+ self.ttls: dict[str, int | None] = {}
+
+ async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
+ self.totals[key] = self.totals.get(key, 0) + value
+ self.ttls[key] = ttl
+ return self.totals[key]
+
+
+class _BrokenCounter:
+ async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
+ raise ConnectionError("redis is down")
+
+
+def _model_info(billing_unit: str, count: int, period: str = "month") -> dict:
+ return {
+ "litellm_provider": "vertex_ai",
+ "mode": "chat",
+ "web_search_billing_unit": billing_unit,
+ "search_context_cost_per_query": {"search_context_size_medium": RATE},
+ "search_free_requests": {"count": count, "period": period, "shared_group": "test-group"},
+ }
+
+
+def _response(queries: int | None) -> ModelResponse:
+ details = PromptTokensDetailsWrapper(web_search_requests=queries) if queries is not None else None
+ return ModelResponse(
+ usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, prompt_tokens_details=details)
+ )
+
+
+def _call(model_info: dict, response_cost: float, cache_hit: bool | None = None) -> tuple[dict, dict]:
+ sl_object = {
+ "response_cost": response_cost,
+ "cache_hit": cache_hit,
+ "cost_breakdown": {"input_cost": 0.1, "output_cost": 0.2, "tool_usage_cost": 0.5, "total_cost": response_cost},
+ "model_map_information": {"model_map_key": "gemini-test", "model_map_value": model_info},
+ }
+ return {"response_cost": response_cost, "standard_logging_object": sl_object}, sl_object
+
+
+@pytest.mark.parametrize(
+ ("allowance", "units", "total_after", "expected"),
+ [
+ (5000, 5, 5, 5),
+ (5000, 5, 5000, 5),
+ (5000, 10, 5005, 5),
+ (5000, 5, 5010, 0),
+ (2, 5, 5, 2),
+ (0, 3, 3, 0),
+ ],
+)
+def test_freed_units_covers_only_what_landed_under_the_allowance(allowance, units, total_after, expected):
+ assert freed_units(allowance, units, total_after) == expected
+
+
+def test_counter_key_is_per_group_and_utc_period():
+ assert search_free_tier_key({"count": 1, "period": "month", "shared_group": "g3"}, NOW) == (
+ "litellm:search_free_tier:g3:2026-10"
+ )
+ assert search_free_tier_key({"count": 1, "period": "day", "shared_group": "g2"}, NOW) == (
+ "litellm:search_free_tier:g2:2026-10-20"
+ )
+
+
+@pytest.mark.parametrize(
+ ("period", "boundary"),
+ [("month", datetime(2026, 11, 1, tzinfo=timezone.utc)), ("day", datetime(2026, 10, 21, tzinfo=timezone.utc))],
+)
+def test_counter_ttl_outlives_the_period_boundary_by_under_two_hours(period, boundary):
+ expiry = NOW + timedelta(seconds=search_free_tier_ttl({"count": 1, "period": period, "shared_group": "g"}, NOW))
+ assert boundary < expiry < boundary + timedelta(hours=2)
+
+
+@pytest.mark.asyncio
+async def test_per_query_allowance_is_shared_across_calls_and_split_on_the_crossing_call():
+ counter = _Counter()
+ model_info = _model_info("per_query", count=7)
+ reduced = []
+ for _ in range(3):
+ kwargs, sl_object = _call(model_info, response_cost=1.0)
+ cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
+ reduced.append((cost, kwargs["response_cost"], sl_object["response_cost"], sl_object["cost_breakdown"]))
+ (c1, k1, s1, b1), (c2, k2, s2, b2), (c3, k3, s3, b3) = reduced
+ assert c1 == k1 == s1 == pytest.approx(1.0 - 5 * RATE)
+ assert b1["original_cost"] == 1.0 and b1["discount_amount"] == pytest.approx(5 * RATE)
+ assert b1["total_cost"] == pytest.approx(c1) and b1["tool_usage_cost"] == 0.5
+ assert c2 == k2 == s2 == pytest.approx(1.0 - 2 * RATE)
+ assert b2["discount_amount"] == pytest.approx(2 * RATE)
+ assert c3 == k3 == s3 == 1.0 and "discount_amount" not in b3
+ assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 15}
+ assert counter.ttls["litellm:search_free_tier:test-group:2026-10"] > 0
+
+
+@pytest.mark.asyncio
+async def test_per_prompt_allowance_counts_a_grounded_prompt_once_whatever_the_query_count():
+ counter = _Counter()
+ model_info = _model_info("per_prompt", count=1, period="day")
+ kwargs, sl_object = _call(model_info, response_cost=1.0)
+ first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
+ kwargs, sl_object = _call(model_info, response_cost=1.0)
+ second = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
+ assert first == pytest.approx(1.0 - RATE)
+ assert second == 1.0
+ assert counter.totals == {"litellm:search_free_tier:test-group:2026-10-20": 2}
+
+
+@pytest.mark.asyncio
+async def test_sidecar_shaped_response_mapping_is_counted_like_a_model_response():
+ counter = _Counter()
+ kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
+ response = {
+ "usage": {
+ "prompt_tokens": 1,
+ "completion_tokens": 1,
+ "total_tokens": 2,
+ "prompt_tokens_details": {"web_search_requests": 3},
+ }
+ }
+ assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == pytest.approx(1.0 - 3 * RATE)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("counter", [None, _BrokenCounter()])
+async def test_full_price_when_no_counter_is_available(counter):
+ kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
+ assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0
+ assert kwargs["response_cost"] == 1.0 and "discount_amount" not in sl_object["cost_breakdown"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("response", [_response(0), _response(None), ModelResponse()])
+async def test_requests_without_grounding_never_touch_the_counter(response):
+ counter = _Counter()
+ kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
+ assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == 1.0
+ assert counter.totals == {}
+
+
+@pytest.mark.asyncio
+async def test_models_without_an_allowance_never_touch_the_counter():
+ counter = _Counter()
+ model_info = {k: v for k, v in _model_info("per_query", count=100).items() if k != "search_free_requests"}
+ kwargs, sl_object = _call(model_info, response_cost=1.0)
+ assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0
+ assert counter.totals == {}
+
+
+@pytest.mark.asyncio
+async def test_a_second_callback_run_on_the_same_request_counts_it_once():
+ counter = _Counter()
+ kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
+ first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
+ second = await apply_search_free_tier(kwargs, sl_object, first, _response(5), counter, NOW)
+ assert first == second == pytest.approx(1.0 - 5 * RATE)
+ assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 5}
+
+
+@pytest.mark.asyncio
+async def test_a_cache_hit_with_grounded_usage_is_not_counted_and_stays_at_zero():
+ counter = _Counter()
+ kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=0.0, cache_hit=True)
+ assert await apply_search_free_tier(kwargs, sl_object, 0.0, _response(2), counter, NOW) == 0.0
+ assert counter.totals == {}
+ assert "discount_amount" not in sl_object["cost_breakdown"]
+
+
+@pytest.mark.asyncio
+async def test_freed_cost_never_exceeds_the_billed_cost():
+ counter = _Counter()
+ billed = 3 * RATE
+ kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=billed)
+ assert await apply_search_free_tier(kwargs, sl_object, billed, _response(5), counter, NOW) == 0.0
+ assert sl_object["cost_breakdown"]["discount_amount"] == pytest.approx(billed)
+ assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 5}
diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py
index 20b46e27213..628870259a4 100644
--- a/tests/unit/test_model_prices_and_context_window.py
+++ b/tests/unit/test_model_prices_and_context_window.py
@@ -15,6 +15,7 @@ from litellm.litellm_core_utils.fallback_generalizations import match_capability
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
+from litellm.types.utils import SearchFreeRequests
REPO_ROOT = Path(__file__).parents[2]
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
@@ -721,6 +722,33 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered():
]
+def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]:
+ prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes())
+ return MappingProxyType(
+ {key: row for key, row in prices.items() if isinstance(row, dict) and "search_free_requests" in row}
+ )
+
+
+@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
+def test_search_free_requests_rows_are_grounding_rows_with_one_allowance_per_group(path: Path) -> None:
+ rows = _search_free_requests_rows(path)
+ assert rows
+ allowances = {
+ key: TypeAdapter(SearchFreeRequests).validate_python(row["search_free_requests"]) for key, row in rows.items()
+ }
+ for key, row in rows.items():
+ assert row.get("supports_web_search") is True, key
+ assert row.get("search_context_cost_per_query"), key
+ assert allowances[key]["count"] > 0, key
+ for group in {allowance["shared_group"] for allowance in allowances.values()}:
+ same_group = {(a["count"], a["period"]) for a in allowances.values() if a["shared_group"] == group}
+ assert len(same_group) == 1, (group, same_group)
+
+
+def test_search_free_requests_backup_rows_match_the_main_catalog() -> None:
+ assert _search_free_requests_rows(BACKUP_PRICES_PATH) == _search_free_requests_rows(PRICES_PATH)
+
+
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(path: Path):
"""Volcengine Ark lists doubao-seed-2-1-pro-260915 at 6 / 30 CNY per million input / output tokens with
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index 40eb5c31fda..a1bdc4d5723 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -1260,6 +1260,16 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"type": "string",
"enum": ["per_prompt", "per_query"],
},
+ "search_free_requests": {
+ "type": "object",
+ "properties": {
+ "count": {"type": "integer", "minimum": 1},
+ "period": {"type": "string", "enum": ["day", "month"]},
+ "shared_group": {"type": "string"},
+ },
+ "required": ["count", "period", "shared_group"],
+ "additionalProperties": False,
+ },
"citation_cost_per_token": {"type": "number"},
"supported_modalities": {
"type": "array",
From 7e8be61fa04c253fb9500b2d146a6b169472d232 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
<158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 20 Oct 2026 18:01:36 +0000
Subject: [PATCH 012/203] test: fix flaky team delete, refusal sink,
loop-blocking, Moyai tamper and model-cost leak tests (#47135)
* test(integration): stabilize auth and team delete checks
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test(unit): make loop-blocking and tampered-signature checks deterministic
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test(unit): isolate model-cost state and synchronize loop progress
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test(unit): fail loop-progress handshakes when the blocking call never starts
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---------
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
.../test_team_delete_large_membership.py | 2 +-
...est_anthropic_refusal_stop_details_wire.py | 6 +-
.../vertex_ai/count_tokens/test_context.py | 58 +++++++++++-------
tests/unit/proxy/test_moyai_endpoints.py | 6 +-
tests/unit/proxy/test_proxy_token_counter.py | 61 +++++++++++--------
tests/unit/test_utils.py | 3 +-
6 files changed, 82 insertions(+), 54 deletions(-)
diff --git a/tests/integration/management/test_team_delete_large_membership.py b/tests/integration/management/test_team_delete_large_membership.py
index 3e4bb568394..e752c835e4f 100644
--- a/tests/integration/management/test_team_delete_large_membership.py
+++ b/tests/integration/management/test_team_delete_large_membership.py
@@ -727,7 +727,7 @@ def _models_status(proxy: Gateway, key: str) -> int:
def _statuses(proxy: Gateway, keys: Sequence[str]) -> dict[int, int]:
- with ThreadPoolExecutor(max_workers=16) as pool:
+ with ThreadPoolExecutor(max_workers=2) as pool:
statuses: Final = tuple(pool.map(lambda key: _models_status(proxy, key), keys))
return {status: statuses.count(status) for status in frozenset(statuses)}
diff --git a/tests/integration/providers/test_anthropic_refusal_stop_details_wire.py b/tests/integration/providers/test_anthropic_refusal_stop_details_wire.py
index b47dd242429..cd42e816ee3 100644
--- a/tests/integration/providers/test_anthropic_refusal_stop_details_wire.py
+++ b/tests/integration/providers/test_anthropic_refusal_stop_details_wire.py
@@ -896,7 +896,7 @@ def test_failing_sinks_get_one_attempt_each_and_the_proxy_stays_healthy(rig: Rig
assert (readiness.status_code, readiness.json()["status"]) == (200, "healthy"), readiness.text
-def test_unauthenticated_refusal_request_never_reaches_the_upstream_or_the_sink(rig: Rig) -> None:
+def test_unauthenticated_refusal_request_never_reaches_upstream_and_logs_failure(rig: Rig) -> None:
with rig.gateway.scenario() as scenario:
model: Final = _deployment(scenario, "anthropic", rig)
stranger: Final = "sk-not-a-virtual-key-" + _marker()
@@ -908,7 +908,9 @@ def test_unauthenticated_refusal_request_never_reaches_the_upstream_or_the_sink(
)
assert response.status_code == 401, response.text
assert _targets(rig.wire) == ()
- assert rig.sink.for_key(stranger) == ()
+ failures: Final = rig.sink.await_key(stranger)
+ assert len(failures) == 1, failures
+ assert failures[0]["status"] == "failure", failures[0]
@pytest.mark.parametrize("fixture", ["null", "missing"])
diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py
index 948f66ab84d..a1643f0f699 100644
--- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py
+++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py
@@ -1,12 +1,12 @@
import asyncio
import base64
import logging
-import time
import uuid
from collections.abc import Generator
from contextlib import contextmanager
+from dataclasses import dataclass, field
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from threading import Thread
+from threading import Event, Thread
from typing import Final, Literal
from unittest.mock import patch
@@ -297,25 +297,37 @@ async def test_openai_messages_keep_their_chat_conversion(custom_llm_provider: L
_PNG_BYTES: Final = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
)
-_DOWNLOAD_SECONDS: Final = 0.5
+_LOOP_PROOF_TIMEOUT_SECONDS: Final = 5
-class _SlowPngHandler(BaseHTTPRequestHandler):
- def do_GET(self) -> None:
- time.sleep(_DOWNLOAD_SECONDS)
- self.send_response(200)
- self.send_header("Content-Type", "image/png")
- self.send_header("Content-Length", str(len(_PNG_BYTES)))
- self.end_headers()
- self.wfile.write(_PNG_BYTES)
+@dataclass(frozen=True, slots=True)
+class _LoopProbe:
+ request_arrived: Event = field(default_factory=Event)
+ loop_ran: Event = field(default_factory=Event)
+ loop_ran_while_download_pending: Event = field(default_factory=Event)
- def log_message(self, format: str, *args: object) -> None:
- return None
+
+def _png_handler_waiting_for_loop(probe: _LoopProbe) -> type[BaseHTTPRequestHandler]:
+ class _PngHandler(BaseHTTPRequestHandler):
+ def do_GET(self) -> None:
+ probe.request_arrived.set()
+ if probe.loop_ran.wait(_LOOP_PROOF_TIMEOUT_SECONDS):
+ probe.loop_ran_while_download_pending.set()
+ self.send_response(200)
+ self.send_header("Content-Type", "image/png")
+ self.send_header("Content-Length", str(len(_PNG_BYTES)))
+ self.end_headers()
+ self.wfile.write(_PNG_BYTES)
+
+ def log_message(self, format: str, *args: object) -> None:
+ return None
+
+ return _PngHandler
@contextmanager
-def _slow_png_server() -> Generator[str, None, None]:
- with ThreadingHTTPServer(("127.0.0.1", 0), _SlowPngHandler) as server:
+def _png_server_waiting_for_loop(probe: _LoopProbe) -> Generator[str, None, None]:
+ with ThreadingHTTPServer(("127.0.0.1", 0), _png_handler_waiting_for_loop(probe)) as server:
thread: Final = Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
@@ -324,10 +336,9 @@ def _slow_png_server() -> Generator[str, None, None]:
server.shutdown()
-async def _event_loop_wake_delay(nap_seconds: float) -> float:
- started: Final = time.monotonic()
- await asyncio.sleep(nap_seconds)
- return time.monotonic() - started
+async def _run_loop_once_download_starts(probe: _LoopProbe) -> None:
+ assert await asyncio.to_thread(probe.request_arrived.wait, _LOOP_PROOF_TIMEOUT_SECONDS)
+ probe.loop_ran.set()
@pytest.mark.parametrize("custom_llm_provider", ("vertex_ai", "gemini"))
@@ -335,10 +346,11 @@ async def test_remote_image_downloads_without_blocking_the_event_loop(
custom_llm_provider: Literal["vertex_ai", "gemini"], monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "user_url_validation", False)
- with _slow_png_server() as base_url:
+ probe: Final = _LoopProbe()
+ with _png_server_waiting_for_loop(probe) as base_url:
image_url: Final = f"{base_url}/{uuid.uuid4().hex}.png"
- wake_delay, contents = await asyncio.gather(
- _event_loop_wake_delay(0.05),
+ _, contents = await asyncio.gather(
+ _run_loop_once_download_starts(probe),
gemini_count_contents(
(
{
@@ -365,7 +377,7 @@ async def test_remote_image_downloads_without_blocking_the_event_loop(
],
}
]
- assert wake_delay < _DOWNLOAD_SECONDS / 2
+ assert probe.loop_ran_while_download_pending.is_set()
class _MissingImageHandler(BaseHTTPRequestHandler):
diff --git a/tests/unit/proxy/test_moyai_endpoints.py b/tests/unit/proxy/test_moyai_endpoints.py
index 92a1261d952..aecda75c661 100644
--- a/tests/unit/proxy/test_moyai_endpoints.py
+++ b/tests/unit/proxy/test_moyai_endpoints.py
@@ -186,11 +186,13 @@ async def test_exchange_without_database_fails_before_nonce(monkeypatch: pytest.
@pytest.mark.asyncio
async def test_exchange_rejects_tampered_signature(monkeypatch: pytest.MonkeyPatch) -> None:
from fastapi import HTTPException
- from litellm.proxy.moyai_endpoints import _sign_connect_code
+ from litellm.proxy.moyai_endpoints import _b64url, _b64url_decode, _sign_connect_code
await _exchange_env(monkeypatch)
code = _sign_connect_code("sk-master", "https://moyai.example.com", "admin-user")
- tampered = code[:-2] + ("aa" if not code.endswith("aa") else "bb")
+ payload_b64, signature_b64 = code.split(".", 1)
+ signature = _b64url_decode(signature_b64)
+ tampered = f"{payload_b64}.{_b64url(bytes((signature[0] ^ 1,)) + signature[1:])}"
with pytest.raises(HTTPException) as exc:
await _exchange_call(tampered, "https://moyai.example.com")
diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py
index 46e88614bb6..fa0c9eda02d 100644
--- a/tests/unit/proxy/test_proxy_token_counter.py
+++ b/tests/unit/proxy/test_proxy_token_counter.py
@@ -9,7 +9,6 @@ import itertools
import json
import logging
import threading
-import time
import traceback
from collections.abc import Callable, Coroutine, Mapping
from functools import partial
@@ -2730,7 +2729,7 @@ async def test_slow_cost_map_lookup_does_not_stall_other_requests_on_the_worker(
request=TokenCountRequest(model="slow-lookup-model", messages=[{"role": "user", "content": "hi"}])
)
)
- await asyncio.to_thread(lookup_started.wait, 5)
+ assert await asyncio.to_thread(lookup_started.wait, 5)
other_request_ran.set()
await counting
@@ -2854,17 +2853,24 @@ async def test_shipped_token_count_model_keeps_the_prompt_in_the_deployment_geog
assert (counter.calls[0][0], counter.regions[0]) == ("bedrock_mantle/anthropic.claude-opus-5-5", counted_in)
-class _SlowVault:
+_LOOP_PROGRESS_TIMEOUT_SECONDS: Final = 5
+
+
+class _VaultWaitingForLoopProgress:
+ def __init__(
+ self,
+ lookup_started: threading.Event,
+ loop_progressed: threading.Event,
+ loop_progressed_during_lookup: threading.Event,
+ ) -> None:
+ self._lookup_started = lookup_started
+ self._loop_progressed = loop_progressed
+ self._loop_progressed_during_lookup = loop_progressed_during_lookup
+
def sync_read_secret(self, secret_name: str) -> None:
- time.sleep(0.5)
-
-
-async def _longest_stall(done: asyncio.Event) -> float:
- if done.is_set():
- return 0.0
- started: Final = time.monotonic()
- await asyncio.sleep(0.01)
- return max(time.monotonic() - started, await _longest_stall(done))
+ self._lookup_started.set()
+ if self._loop_progressed.wait(_LOOP_PROGRESS_TIMEOUT_SECONDS):
+ self._loop_progressed_during_lookup.set()
@pytest.mark.asyncio
@@ -2884,25 +2890,30 @@ async def test_region_lookup_for_a_shipped_count_model_leaves_the_event_loop_fre
),
)
monkeypatch.setattr(proxy_server, "_get_provider_token_counter", bedrock_counter)
- monkeypatch.setattr(litellm, "secret_manager_client", _SlowVault())
+ lookup_started: Final = threading.Event()
+ loop_progressed: Final = threading.Event()
+ loop_progressed_during_lookup: Final = threading.Event()
+ monkeypatch.setattr(
+ litellm,
+ "secret_manager_client",
+ _VaultWaitingForLoopProgress(lookup_started, loop_progressed, loop_progressed_during_lookup),
+ )
monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.HASHICORP_VAULT)
monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only"))
- done: Final = asyncio.Event()
async def count() -> TokenCountResponse:
- try:
- return await token_counter(
- request=TokenCountRequest(model="sonnet", messages=[{"role": "user", "content": "hi"}]),
- call_endpoint=True,
- )
- finally:
- done.set()
+ return await token_counter(
+ request=TokenCountRequest(model="sonnet", messages=[{"role": "user", "content": "hi"}]),
+ call_endpoint=True,
+ )
- started: Final = time.monotonic()
- stall, response = await asyncio.gather(_longest_stall(done), count())
+ async def prove_loop_progress() -> None:
+ assert await asyncio.to_thread(lookup_started.wait, _LOOP_PROGRESS_TIMEOUT_SECONDS)
+ loop_progressed.set()
- assert time.monotonic() - started >= 1.0
- assert stall < 0.4
+ response, _ = await asyncio.gather(count(), prove_loop_progress())
+
+ assert loop_progressed_during_lookup.is_set()
assert response.total_tokens == 447
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index a1bdc4d5723..24d3ed678c9 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -4841,7 +4841,8 @@ def test_reapply_runtime_registrations_replays_register_model_overrides(
# installed would make this depend on when that happens.
monkeypatch.setattr(litellm_utils._LiveDeploymentReplay, "callback", None)
- saved_model_cost = litellm.model_cost
+ saved_model_cost: Final = litellm.model_cost
+ monkeypatch.setattr(litellm, "model_cost", dict(saved_model_cost))
try:
litellm.register_model(
model_cost={
From 2773ef0084b85bced936d4174b10b1d70df6ae65 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
<158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 20 Oct 2026 18:11:18 +0000
Subject: [PATCH 013/203] test(integration): remove BUG skip for 1 test that
passes on main (#47603)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
.../compatibility/test_audio_transcriptions_wire.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/tests/integration/compatibility/test_audio_transcriptions_wire.py b/tests/integration/compatibility/test_audio_transcriptions_wire.py
index 8f9a83e9912..3deeaebaf7d 100644
--- a/tests/integration/compatibility/test_audio_transcriptions_wire.py
+++ b/tests/integration/compatibility/test_audio_transcriptions_wire.py
@@ -660,10 +660,6 @@ def test_audio_transcription_preserves_the_client_file_content_type(gateway: Gat
def test_audio_transcription_bills_audio_tokens_at_the_input_rate_when_no_audio_rate_is_set(
gateway: Gateway,
) -> None:
- pytest.skip(
- "BUG: with only input_cost_per_token set, transcription audio tokens are billed at 0, so "
- "x-litellm-response-cost is 1e-05 instead of 2e-05 for 10 audio, 2 text and 4 output tokens"
- )
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
def respond(request: Request) -> Reply:
From 7f0bb8424c5e075e84b18ca726e022ed2fb5be6c Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 11:20:37 -0700
Subject: [PATCH 014/203] feat(proxy): size pass drops the oldest spend log
partitions first (#47451)
* feat(proxy): size pass drops the oldest spend log partitions first
On a range-partitioned LiteLLM_SpendLogs the size pass drops fully
elapsed partitions oldest first, never the current or DEFAULT one,
remeasures after each drop and falls back to oldest-first row deletes
only while still above the target
* fix(proxy): a partition drop rebases the size state instead of reading as a statistics reset
Bugbot on #47449: dropping a partition removes that child's n_tup_ins and
n_tup_del from the summed probe, which advance_size_state took for a reset
and reseeded from n_dead_tup. The pass now saves the counters of the last
post-drop probe as the state's base and scales the reclaimable count to
the surviving files
* style(proxy): keep the mutable-ok reason on the line it suppresses
* fix(proxy): size pass stops when a post-drop re-measure fails and keeps day partitions ensured
A failed storage probe after a partition drop no longer reads as under target. The pass logs a warning, skips the row step and saves no count, so neither live rows get deleted on the pre-drop numbers nor the next check reads the drop as a statistics reset. The partitioned size path also calls ensure_partitions before dropping, so a size-only setup keeps getting day partitions instead of writing into DEFAULT
---------
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
.../db_transaction_queue/spend_log_cleanup.py | 74 ++++++++++--
.../spend_logs_partition_manager.py | 48 +++++++-
tests/unit/proxy/test_spend_log_cleanup.py | 108 ++++++++++++++++++
3 files changed, 219 insertions(+), 11 deletions(-)
diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
index 268f0230d7a..726bc2b221c 100644
--- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
+++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
@@ -2,7 +2,7 @@ import asyncio
import math
import re
import time
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
from contextvars import ContextVar
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
@@ -254,6 +254,7 @@ class SizePassReport:
rule: SizeRule
before: SpendLogStorage
after: SpendLogStorage
+ partitions_dropped: tuple[str, ...]
results: tuple[TableCleanupResult, ...]
@property
@@ -1028,7 +1029,9 @@ class SpendLogCleanup:
Bring the spend-log tables under the size target, oldest rows first.
Live bytes are the files on disk less the reclaimable count carried in LiteLLM_Config (see
- `SizeCleanupState`), rolled forward from the catalog's cumulative insert and delete counters. Rows go in steps: the rows still needed to reach the target follow
+ `SizeCleanupState`), rolled forward from the catalog's cumulative insert and delete counters. A
+ partitioned table gives its oldest fully elapsed partitions back first, because a DROP returns disk at
+ once, re-measuring after each. Then rows go in steps: the rows still needed to reach the target follow
from bytes per row, and each step turns that many of the oldest rows (at most
SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES batches) into a cutoff and runs the same bounded DELETE the age
rule uses, for the spend logs and the tool index rows derived from them. Progress is counted from the
@@ -1066,7 +1069,7 @@ class SpendLogCleanup:
)
SpendLogCleanupMetrics.set_storage_bytes("above_target", 0)
await self._save_size_state(prisma_client, deadline, state)
- return SizePassReport(rule=rule, before=before, after=before, results=())
+ return SizePassReport(rule=rule, before=before, after=before, partitions_dropped=(), results=())
verbose_proxy_logger.warning(
"Spend-log storage is over maximum_spend_logs_size (estimated_live=%s allocated=%s reclaimable=%s "
"limit=%s bytes); deleting the oldest spend logs down to %s bytes",
@@ -1076,9 +1079,40 @@ class SpendLogCleanup:
rule.limit_bytes,
rule.target_bytes,
)
- rows_to_delete: Final = max(
- 0, math.ceil((before.estimated_live_bytes - rule.target_bytes) / before.bytes_per_row)
+ latest: Final[list[SpendLogStorage]] = [before] # mutable-ok: last measurement, rewritten per partition drop
+ baseline: Final[list[StorageProbe]] = [probe] # mutable-ok: counters after the last drop
+
+ remeasure_failed: Final[list[bool]] = [False] # mutable-ok: set when a post-drop probe fails
+
+ async def still_over_target() -> bool:
+ measured: Final = await self._probe_storage(prisma_client, deadline)
+ if measured is None:
+ remeasure_failed[0] = True
+ return False
+ kept: Final = min(1.0, measured.allocated_bytes / max(latest[0].allocated_bytes, 1))
+ latest[0] = SpendLogStorage(
+ allocated_bytes=measured.allocated_bytes,
+ reclaimable_bytes=min(measured.allocated_bytes, round(latest[0].reclaimable_bytes * kept)),
+ bytes_per_row=latest[0].bytes_per_row,
+ live_rows=measured.live_rows,
+ )
+ baseline[0] = measured
+ return latest[0].estimated_live_bytes > rule.target_bytes
+
+ partitions_dropped: Final = await self._drop_oldest_partitions_for_size(
+ prisma_client, deadline, still_over_target
)
+ if remeasure_failed[0]:
+ verbose_proxy_logger.warning(
+ "Spend-log storage could not be re-measured after dropping partitions %s; the row step and the saved "
+ "count wait for the next check rather than run on the pre-drop numbers",
+ partitions_dropped,
+ )
+ return SizePassReport(
+ rule=rule, before=before, after=latest[0], partitions_dropped=partitions_dropped, results=()
+ )
+ base: Final = latest[0]
+ rows_to_delete: Final = max(0, math.ceil((base.estimated_live_bytes - rule.target_bytes) / base.bytes_per_row))
def rows_still_needed(rows_deleted: int) -> int:
return rows_to_delete - rows_deleted
@@ -1086,20 +1120,39 @@ class SpendLogCleanup:
results: Final = await self._delete_oldest_rows_for_size(prisma_client, deadline, rows_still_needed)
rows_deleted: Final = sum(result.rows_deleted for result in results)
remeasured: Final = await self._probe_storage(prisma_client, deadline)
- after: Final = before.after_deleting(
- rows_deleted, remeasured.allocated_bytes if remeasured is not None else before.allocated_bytes
+ after: Final = base.after_deleting(
+ rows_deleted, remeasured.allocated_bytes if remeasured is not None else base.allocated_bytes
)
await self._save_size_state(
prisma_client,
deadline,
- after.to_state(inserted_total=probe.inserted_total, deleted_total=probe.deleted_total + rows_deleted),
+ after.to_state(
+ inserted_total=baseline[0].inserted_total, deleted_total=baseline[0].deleted_total + rows_deleted
+ ),
+ )
+ report: Final = SizePassReport(
+ rule=rule, before=before, after=after, partitions_dropped=partitions_dropped, results=results
)
- report: Final = SizePassReport(rule=rule, before=before, after=after, results=results)
SpendLogCleanupMetrics.set_storage_bytes("allocated", after.allocated_bytes)
SpendLogCleanupMetrics.set_storage_bytes("estimated_live", after.estimated_live_bytes)
SpendLogCleanupMetrics.set_storage_bytes("above_target", report.above_target_bytes)
return report
+ async def _drop_oldest_partitions_for_size(
+ self, prisma_client: PrismaClient, deadline: float, still_over_target: Callable[[], Awaitable[bool]]
+ ) -> tuple[str, ...]:
+ remaining_timeout_ms: Final = self._remaining_timeout_ms(deadline)
+ if time.monotonic() >= deadline or not self._setting("use_spend_logs_partitioning"):
+ return ()
+ if not await self.partition_manager.is_partitioned(prisma_client, remaining_timeout_ms):
+ return ()
+ await self.partition_manager.ensure_partitions(prisma_client, remaining_timeout_ms)
+ dropped: Final = await self.partition_manager.drop_oldest_partitions_while(
+ prisma_client, datetime.now(timezone.utc), remaining_timeout_ms, still_over_target
+ )
+ verbose_proxy_logger.info("Dropped %d oldest spend-log partitions for the size rule: %s", len(dropped), dropped)
+ return tuple(dropped)
+
async def _delete_oldest_rows_for_size(
self,
prisma_client: PrismaClient,
@@ -1275,7 +1328,8 @@ class SpendLogCleanup:
f"allocated_after={report.after.allocated_bytes} estimated_live_after={report.after.estimated_live_bytes} "
f"reclaimable_after={report.after.reclaimable_bytes} "
f"limit={report.rule.limit_bytes} target={report.rule.target_bytes} "
- f"target_reached={str(report.target_reached).lower()} above_target={report.above_target_bytes}{reclaim_note}"
+ f"target_reached={str(report.target_reached).lower()} above_target={report.above_target_bytes} "
+ f"partitions_dropped={len(report.partitions_dropped)}{reclaim_note}"
)
@classmethod
diff --git a/litellm/proxy/db/db_transaction_queue/spend_logs_partition_manager.py b/litellm/proxy/db/db_transaction_queue/spend_logs_partition_manager.py
index 3f9384d5d15..049a6ecbeee 100644
--- a/litellm/proxy/db/db_transaction_queue/spend_logs_partition_manager.py
+++ b/litellm/proxy/db/db_transaction_queue/spend_logs_partition_manager.py
@@ -14,7 +14,7 @@ keeps the batched-DELETE path, so existing deployments are untouched.
"""
import re
-from collections.abc import Callable
+from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
from datetime import date, datetime, timedelta, timezone
from typing import (
TYPE_CHECKING,
@@ -265,6 +265,52 @@ class SpendLogsPartitionManager:
)
return [(row["name"], parse_partition_upper_bound(row.get("bound") or "")) for row in rows]
+ async def drop_oldest_partitions_while(
+ self,
+ prisma_client: "PrismaClient",
+ now: datetime,
+ remaining_timeout_ms: RemainingTimeoutMs,
+ still_over_target: Callable[[], Awaitable[bool]],
+ ) -> tuple[str, ...]:
+ """
+ DROP fully elapsed partitions oldest first until `still_over_target` says the size rule is satisfied.
+
+ Only partitions whose whole range lies before `now` are candidates, so the partition receiving
+ current writes and the DEFAULT partition are never dropped. The callback runs after every drop
+ because a DROP returns its disk at once, so a re-measure is what tells the loop to stop.
+ """
+ list_budget_ms: Final = remaining_timeout_ms()
+ if list_budget_ms is None:
+ return ()
+ now_naive: Final = now.astimezone(timezone.utc).replace(tzinfo=None)
+ partitions: Final = await self._list_partitions(prisma_client, list_budget_ms)
+ elapsed: Final = sorted(
+ ((upper, name) for name, upper in partitions if upper is not None and upper <= now_naive),
+ )
+ drops: Final = self._drop_each_while(prisma_client, elapsed, remaining_timeout_ms, still_over_target)
+ return tuple([name async for name in drops])
+
+ async def _drop_each_while(
+ self,
+ prisma_client: "PrismaClient",
+ elapsed: Sequence[tuple[datetime, str]],
+ remaining_timeout_ms: RemainingTimeoutMs,
+ still_over_target: Callable[[], Awaitable[bool]],
+ ) -> AsyncIterator[str]:
+ for _, name in elapsed:
+ budget_ms: int | None = remaining_timeout_ms()
+ if budget_ms is None:
+ verbose_proxy_logger.info("Run budget spent, leaving the remaining partitions for the next run")
+ return
+ try:
+ await self._execute_bounded_ddl(prisma_client, f'DROP TABLE IF EXISTS "{name}"', budget_ms)
+ except Exception as e: # noqa: BLE001 # a partition this run cannot drop is left for the next one
+ verbose_proxy_logger.warning("Failed to drop spend-log partition %s: %s", name, e)
+ return
+ yield name
+ if not await still_over_target():
+ return
+
async def drop_partitions_older_than(
self, prisma_client: "PrismaClient", cutoff: datetime, remaining_timeout_ms: RemainingTimeoutMs
) -> list[str]:
diff --git a/tests/unit/proxy/test_spend_log_cleanup.py b/tests/unit/proxy/test_spend_log_cleanup.py
index e70b93f7955..8c97f829025 100644
--- a/tests/unit/proxy/test_spend_log_cleanup.py
+++ b/tests/unit/proxy/test_spend_log_cleanup.py
@@ -1998,6 +1998,114 @@ async def test_the_retention_run_carries_the_size_pass_only_when_one_cron_drives
assert measured is size_pass_expected
+@pytest.mark.asyncio
+async def test_partitioned_size_pass_drops_the_oldest_partitions_first_and_skips_row_deletes_once_under_target(
+ caplog,
+):
+ db = MagicMock()
+ _wire_tx(db)
+ db.execute_raw = AsyncMock(return_value=1000)
+ saves = _wire_size_probes(
+ db,
+ [_storage(10 * 1024**3, live=1000, dead=0), _storage(256 * 1024**2, live=25, dead=0)],
+ cutoff=datetime.now(timezone.utc),
+ )
+ client = MagicMock()
+ client.db = db
+ verdicts = []
+
+ async def _drop(prisma_client, now, remaining_timeout_ms, still_over_target):
+ verdicts.append(await still_over_target())
+ return ("LiteLLM_SpendLogs_p20260601",)
+
+ partition_manager = MagicMock()
+ partition_manager.is_partitioned = AsyncMock(return_value=True)
+ partition_manager.ensure_partitions = AsyncMock()
+ partition_manager.drop_oldest_partitions_while = AsyncMock(side_effect=_drop)
+ cleaner = _size_cleaner(
+ {"use_spend_logs_partitioning": True, "maximum_spend_logs_cleanup_target_size": "512MB"},
+ partition_manager=partition_manager,
+ )
+
+ with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"):
+ await cleaner.cleanup_spend_logs_by_size(client)
+
+ partition_manager.ensure_partitions.assert_awaited_once(), "a size-only setup still needs tomorrow's partition"
+ partition_manager.drop_oldest_partitions_while.assert_awaited_once()
+ assert verdicts == [False]
+ assert db.deletes.await_count == 0
+ summary = next(r.getMessage() for r in caplog.records if "Spend log cleanup run finished" in r.getMessage())
+ assert "partitions_dropped=1" in summary
+ assert "target_reached=true" in summary
+ assert (saves[-1].inserted_total, saves[-1].deleted_total, saves[-1].allocated_bytes) == (25, 0, 256 * 1024**2), (
+ "the saved counters must be the post-drop ones, or the next check reads the drop as a statistics reset"
+ )
+
+
+@pytest.mark.asyncio
+async def test_a_failed_remeasure_after_a_partition_drop_stops_the_pass_instead_of_deleting_on_old_numbers(caplog):
+ db = MagicMock()
+ _wire_tx(db)
+ db.execute_raw = AsyncMock(return_value=1000)
+ saves = _wire_size_probes(db, [_storage(10 * 1024**3, live=1000, dead=0)], cutoff=datetime.now(timezone.utc))
+ client = MagicMock()
+ client.db = db
+
+ async def _drop(prisma_client, now, remaining_timeout_ms, still_over_target):
+ db.query_raw = AsyncMock(side_effect=RuntimeError("connection reset"))
+ assert await still_over_target() is False
+ return ("LiteLLM_SpendLogs_p20260601",)
+
+ partition_manager = MagicMock()
+ partition_manager.is_partitioned = AsyncMock(return_value=True)
+ partition_manager.ensure_partitions = AsyncMock()
+ partition_manager.drop_oldest_partitions_while = AsyncMock(side_effect=_drop)
+ cleaner = _size_cleaner(
+ {"use_spend_logs_partitioning": True, "maximum_spend_logs_cleanup_target_size": "512MB"},
+ partition_manager=partition_manager,
+ )
+
+ with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+ await cleaner.cleanup_spend_logs_by_size(client)
+
+ assert db.deletes.await_count == 0, "the row step ran on the pre-drop measurement"
+ assert saves == [], "a count saved from pre-drop counters reads the drop as a statistics reset next check"
+ assert any("could not be re-measured after dropping partitions" in r.getMessage() for r in caplog.records)
+
+
+def test_drop_oldest_partitions_while_only_drops_fully_elapsed_partitions_oldest_first():
+ from litellm.proxy.db.db_transaction_queue.spend_logs_partition_manager import SpendLogsPartitionManager
+
+ manager = SpendLogsPartitionManager()
+ now = datetime(2026, 10, 17, 12, tzinfo=timezone.utc)
+ manager._list_partitions = AsyncMock(
+ return_value=[
+ ("LiteLLM_SpendLogs_p20261017", datetime(2026, 10, 18)), # today's, still receiving writes
+ ("LiteLLM_SpendLogs_p20261015", datetime(2026, 10, 16)),
+ ("LiteLLM_SpendLogs_default", None),
+ ("LiteLLM_SpendLogs_p20261014", datetime(2026, 10, 15)),
+ ]
+ )
+ dropped_sql: list[str] = []
+
+ async def _ddl(prisma_client, statement, timeout_ms):
+ dropped_sql.append(statement)
+
+ manager._execute_bounded_ddl = _ddl
+ verdicts = iter([True, False])
+
+ async def still_over_target():
+ return next(verdicts)
+
+ dropped = asyncio.run(manager.drop_oldest_partitions_while(MagicMock(), now, lambda: 1000, still_over_target))
+
+ assert dropped == ("LiteLLM_SpendLogs_p20261014", "LiteLLM_SpendLogs_p20261015")
+ assert dropped_sql == [
+ 'DROP TABLE IF EXISTS "LiteLLM_SpendLogs_p20261014"',
+ 'DROP TABLE IF EXISTS "LiteLLM_SpendLogs_p20261015"',
+ ]
+
+
@pytest.mark.parametrize(
("settings", "retention_job", "size_job"),
[
From e78f7b4602c15a5328f6f211c4bc9f3d664e1d50 Mon Sep 17 00:00:00 2001
From: yuneng-jiang
Date: Tue, 20 Oct 2026 11:21:04 -0700
Subject: [PATCH 015/203] test(integration): repair seven stale integration
tests on main (#47600)
The Gemini TTS billing and text completion guardrail tests read spend rows from the owned proxy's own database, which #47213 made private. The cursor wire test derives its effort levels from the gpt-5 cost map row that #43935 now enforces, and the Azure effort check accepts the unquoted level. The pin TTL test asserts the one hour pin #46411 writes for a tool-only mark, the video callback test decodes ids with the product decoder after #44614, and the two Gemini 3.8 flash stream cost cases bill audio at the text rate their rows imply
---
.../cost_calculation/cases/gemini.py | 2 +-
.../cost_calculation/cases/vertex_ai.py | 2 +-
.../observability/test_callback_delivery.py | 8 +++----
.../test_azure_reasoning_effort_none_wire.py | 3 ++-
.../providers/test_cursor_responses_wire.py | 13 +++++++++--
.../routing/test_prompt_cache_pin_ttl.py | 8 ++++---
.../spend/test_gemini_tts_speech_billing.py | 23 ++++++++++---------
...ext_completion_stream_guardrail_rebuild.py | 3 +--
8 files changed, 37 insertions(+), 25 deletions(-)
diff --git a/tests/integration/cost_calculation/cases/gemini.py b/tests/integration/cost_calculation/cases/gemini.py
index 8a50e91ec01..86a0d3b9067 100644
--- a/tests/integration/cost_calculation/cases/gemini.py
+++ b/tests/integration/cost_calculation/cases/gemini.py
@@ -2658,7 +2658,7 @@ GEMINI_GEMINI_3_8_FLASH_STREAM_FULL_USAGE: Final = CostTrackingTestCase(
),
),
expected=ExactExpected(
- spend=0.0076232, input_cost=0.0015572, output_cost=0.006066, prompt_tokens=8314, completion_tokens=1592
+ spend=0.0066182, input_cost=0.0013922, output_cost=0.005226, prompt_tokens=8314, completion_tokens=1592
),
)
diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py
index 305c10535b7..2fb005dbeca 100644
--- a/tests/integration/cost_calculation/cases/vertex_ai.py
+++ b/tests/integration/cost_calculation/cases/vertex_ai.py
@@ -2846,7 +2846,7 @@ GEMINI_3_8_FLASH_STREAM_FULL_USAGE: Final = CostTrackingTestCase(
),
),
expected=ExactExpected(
- spend=0.005775328, input_cost=0.001276288, output_cost=0.00449904, prompt_tokens=8314, completion_tokens=1592
+ spend=0.005946928, input_cost=0.001447888, output_cost=0.00449904, prompt_tokens=8314, completion_tokens=1592
),
)
diff --git a/tests/integration/observability/test_callback_delivery.py b/tests/integration/observability/test_callback_delivery.py
index f3e8daab014..17ba6419646 100644
--- a/tests/integration/observability/test_callback_delivery.py
+++ b/tests/integration/observability/test_callback_delivery.py
@@ -1,4 +1,3 @@
-import base64
import json
import uuid
from collections.abc import Callable, Mapping
@@ -14,6 +13,8 @@ from integration._support.database import read_rows
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.wire import Reply, Request, wire_server
+from litellm.types.videos.utils import decode_video_id_with_provider
+
pytestmark: Final = pytest.mark.shares_worker_state
@@ -431,9 +432,8 @@ def _responses_text(body: dict[str, JsonValue]) -> str:
def _video_job(body: dict[str, JsonValue]) -> tuple[str, str]:
- encoded_id: Final = string_value(body["id"]).removeprefix("video_")
- decoded: Final = base64.b64decode(encoded_id).decode()
- return decoded.rsplit("video_id:", 1)[-1], string_value(body["status"])
+ decoded: Final = decode_video_id_with_provider(string_value(body["id"]))
+ return string_value(decoded.get("video_id")), string_value(body["status"])
@dataclass(frozen=True, slots=True)
diff --git a/tests/integration/providers/test_azure_reasoning_effort_none_wire.py b/tests/integration/providers/test_azure_reasoning_effort_none_wire.py
index c95e497c672..cec50c5ea47 100644
--- a/tests/integration/providers/test_azure_reasoning_effort_none_wire.py
+++ b/tests/integration/providers/test_azure_reasoning_effort_none_wire.py
@@ -1,3 +1,4 @@
+import re
from typing import Final
from integration._support.client import Gateway
@@ -30,7 +31,7 @@ def test_reasoning_effort_none_is_rejected_by_litellm_before_reaching_azure(gate
)
assert response.status_code == 400, response.text
- assert "reasoning_effort='none'" in response.text, response.text
+ assert re.search(r"reasoning_effort='?none\b", response.text), response.text
assert wire.drain() == ()
diff --git a/tests/integration/providers/test_cursor_responses_wire.py b/tests/integration/providers/test_cursor_responses_wire.py
index 6e8c877f7ca..d77d35f2ff8 100644
--- a/tests/integration/providers/test_cursor_responses_wire.py
+++ b/tests/integration/providers/test_cursor_responses_wire.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import json
import os
import uuid
+from importlib.resources import files
from typing import Final
import pytest
@@ -16,8 +17,16 @@ _MODEL: Final = "gpt-5"
_API_KEY: Final = "synthetic-cursor-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1")))
-_THINKING_LEVELS: Final = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
-_CHAT_THINKING_LEVELS: Final = ("none", "minimal", "low", "medium", "high", "max")
+_COST_MAP: Final = TypeAdapter(dict[str, dict[str, JsonValue]])
+_MODEL_ROW: Final = _COST_MAP.validate_json(
+ (files("litellm") / "model_prices_and_context_window_backup.json").read_bytes()
+)[_MODEL]
+_THINKING_LEVELS: Final = tuple(
+ level
+ for level in ("none", "minimal", "low", "medium", "high", "xhigh", "max")
+ if _MODEL_ROW.get(f"supports_{level}_reasoning_effort") is not False
+)
+_CHAT_THINKING_LEVELS: Final = tuple(level for level in _THINKING_LEVELS if level != "xhigh")
_RESPONSES_SUFFIXES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = tuple(
(f"-thinking-{level}", {"reasoning": {"effort": level}}) for level in _THINKING_LEVELS
) + (
diff --git a/tests/integration/routing/test_prompt_cache_pin_ttl.py b/tests/integration/routing/test_prompt_cache_pin_ttl.py
index 0a758d51b7e..f6ee96a9206 100644
--- a/tests/integration/routing/test_prompt_cache_pin_ttl.py
+++ b/tests/integration/routing/test_prompt_cache_pin_ttl.py
@@ -270,7 +270,7 @@ def test_chat_pin_lifetime_follows_the_longest_breakpoint_ttl_in_the_cacheable_p
spend_row_lands(response_id)
-def test_chat_one_hour_mark_on_a_tool_definition_alone_writes_no_pin(rig: _Rig) -> None:
+def test_chat_one_hour_mark_on_a_tool_definition_alone_pins_the_served_deployment_for_an_hour(rig: _Rig) -> None:
before: Final = _before(rig)
marker: Final = new_marker()
tool: Final[dict[str, JsonValue]] = {
@@ -281,8 +281,10 @@ def test_chat_one_hour_mark_on_a_tool_definition_alone_writes_no_pin(rig: _Rig)
response_id, served = post_chat(
rig.owned.gateway, chat_body(_CLAUDE, _first_turn(new_marker(), marker, None), tools=[tool])
)
- assert served == rig.served_by(marker), served
- no_pin_written(rig.redis, before, response_id)
+ _, pin = one_new_pin(rig.redis, before)
+ assert served == rig.served_by(marker) == pin.model_id, (served, pin)
+ assert lives_one_hour(pin), pin
+ spend_row_lands(response_id)
def _openai_client(rig: _Rig) -> openai.OpenAI:
diff --git a/tests/integration/spend/test_gemini_tts_speech_billing.py b/tests/integration/spend/test_gemini_tts_speech_billing.py
index f1bf3de91b3..ce0752beee3 100644
--- a/tests/integration/spend/test_gemini_tts_speech_billing.py
+++ b/tests/integration/spend/test_gemini_tts_speech_billing.py
@@ -11,7 +11,6 @@ from typing import Final, Literal
import pytest
import yaml
from integration._support.client import JSON_OBJECT, eventually, gateway_from_environment, object_value
-from integration._support.database import read_rows
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
from integration._support.vertex import service_account_json
from integration._support.wire import Wire, wire_server
@@ -201,13 +200,15 @@ def _upstream_calls(wire: Wire, text: str) -> int:
return sum(1 for request in wire.drain() if received_text(request) == text)
-def _spend_rows(key: str, count: int) -> list[dict[str, JsonValue]]:
- return eventually(lambda: read_rows(SPEND_ROWS_SQL, (_hashed(key),)), lambda rows: len(rows) == count, seconds=70)
+def _spend_rows(billing: BillingProxy, key: str, count: int) -> list[dict[str, JsonValue]]:
+ return eventually(
+ lambda: billing.proxy.gateway.rows(SPEND_ROWS_SQL, (_hashed(key),)), lambda rows: len(rows) == count, seconds=70
+ )
-def _key_spend(key: str, expected: float) -> JsonValue:
+def _key_spend(billing: BillingProxy, key: str, expected: float) -> JsonValue:
rows: Final = eventually(
- lambda: read_rows(KEY_SPEND_SQL, (_hashed(key),)),
+ lambda: billing.proxy.gateway.rows(KEY_SPEND_SQL, (_hashed(key),)),
lambda found: len(found) == 1 and found[0]["spend"] == pytest.approx(expected),
seconds=70,
return_last_on_timeout=True,
@@ -224,7 +225,7 @@ def _chat_spend(billing: BillingProxy, model: str, key: str) -> float:
}
response: Final = billing.proxy.gateway.request("POST", "/v1/chat/completions", body, key=key)
assert response.status_code == 200, response.text
- row: Final = _spend_rows(key, 1)[0]
+ row: Final = _spend_rows(billing, key, 1)[0]
assert (row["prompt_tokens"], row["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), row
spend: Final = row["spend"]
assert isinstance(spend, float) and spend > 0, row
@@ -261,11 +262,11 @@ def test_speech_calls_google_once_and_bills_and_logs_its_token_usage_once(
assert wav_payload(sent.body) == PCM
assert _upstream_calls(tts_wire, text) == 1
chat_spend: Final = _chat_spend(uncached, MODELS[provider], scenario.key())
- row: Final = _spend_rows(speech_key, 1)[0]
+ row: Final = _spend_rows(uncached, speech_key, 1)[0]
assert row["call_type"] == "aspeech", row
assert (row["prompt_tokens"], row["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), row
assert row["spend"] == pytest.approx(chat_spend), row
- assert _key_spend(speech_key, chat_spend) == pytest.approx(chat_spend)
+ assert _key_spend(uncached, speech_key, chat_spend) == pytest.approx(chat_spend)
events: Final = _speech_events(uncached, speech_key, 1)
assert [_summary(event) for event in events] == [("aspeech", BINARY_RESPONSE, PROMPT_TOKENS, AUDIO_TOKENS)]
assert events[0]["response_cost"] == pytest.approx(chat_spend), events
@@ -286,11 +287,11 @@ def test_cached_speech_repeat_is_a_free_cache_hit_after_a_billed_first_call(
assert _upstream_calls(tts_wire, text) == 1
assert repeat.cache_key, "the cached repeat carried no x-litellm-cache-key header"
chat_spend: Final = _chat_spend(cached, model, scenario.key())
- billed, hit = _spend_rows(speech_key, 2)
+ billed, hit = _spend_rows(cached, speech_key, 2)
assert billed["cache_hit"] != "True" and billed["spend"] == pytest.approx(chat_spend), billed
assert (billed["prompt_tokens"], billed["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), billed
assert (hit["cache_hit"], hit["spend"]) == ("True", 0.0), hit
- assert _key_spend(speech_key, chat_spend) == pytest.approx(chat_spend)
+ assert _key_spend(cached, speech_key, chat_spend) == pytest.approx(chat_spend)
events: Final = _speech_events(cached, speech_key, 2)
assert len(events) == 2, events
hits: Final = tuple(event for event in events if event["cache_hit"] is True)
@@ -325,7 +326,7 @@ def test_a_speech_reply_fills_the_deployment_tpm_limit_so_the_next_request_gets_
first_text: Final = marker()
first: Final = _speech(uncached, TPM_MODEL, first_text, key)
assert first.status == 200, first.body[:300]
- _spend_rows(key, 1)
+ _spend_rows(uncached, key, 1)
second_text: Final = marker()
second: Final = _speech(uncached, TPM_MODEL, second_text, key)
assert second.status == 429, second.body[:300]
diff --git a/tests/integration/streaming/test_text_completion_stream_guardrail_rebuild.py b/tests/integration/streaming/test_text_completion_stream_guardrail_rebuild.py
index 60a33d46b29..229a9be6d20 100644
--- a/tests/integration/streaming/test_text_completion_stream_guardrail_rebuild.py
+++ b/tests/integration/streaming/test_text_completion_stream_guardrail_rebuild.py
@@ -16,7 +16,6 @@ from integration._support.client import (
list_value,
object_value,
)
-from integration._support.database import read_rows
from integration._support.process import OwnedProxy, WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue
@@ -202,7 +201,7 @@ def test_guarded_text_completion_stream_reaches_the_caller_whole_with_its_usage(
], stream.frames
assert _posted_targets(rig.wire) == ("/v1/completions",)
(row,) = eventually(
- lambda: read_rows(
+ lambda: rig.proxy.gateway.rows(
'SELECT prompt_tokens, completion_tokens, metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s',
(identity,),
),
From 3f415cb145f022418a3ec373f9dbdd3953a0cedd Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 18:33:01 +0000
Subject: [PATCH 016/203] docs: clarify unit test clock rule (#47604)
---
tests/unit/AGENTS.md | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/tests/unit/AGENTS.md b/tests/unit/AGENTS.md
index 80a3a87f959..9ce6c58cff7 100644
--- a/tests/unit/AGENTS.md
+++ b/tests/unit/AGENTS.md
@@ -1,6 +1,9 @@
# tests/unit
-In-process. No network, clock or subprocess
+In-process. No network or subprocess
+
+Unit tests must not depend on the current time or execution speed. Use fixed timestamps for fixtures
+and controlled clocks for time-dependent behavior. Incidental clock reads inside production code are allowed
## What good looks like
From 06d56eb6d7bd835b4dcae468c0bee67dedaef0f7 Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 11:37:04 -0700
Subject: [PATCH 017/203] fix(proxy): one spend log cleanup pass at a time per
pod, with or without Redis (#47500)
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
.../db_transaction_queue/spend_log_cleanup.py | 14 ++++++++
tests/unit/proxy/test_spend_log_cleanup.py | 33 ++++++++++++-------
2 files changed, 35 insertions(+), 12 deletions(-)
diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
index 726bc2b221c..e1e1d3690ff 100644
--- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
+++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
@@ -354,6 +354,9 @@ _STORAGE_SQL: Final = """
)
"""
+# The age job and the size job are scheduled on separate SpendLogCleanup instances, so the lock is per process.
+_PASS_LOCK: Final = asyncio.Lock()
+
SPEND_LOG_CLEANUP_BOUND_SETTINGS: Final = (
"maximum_spend_logs_cleanup_batch_size",
"maximum_spend_logs_cleanup_max_batches",
@@ -1363,6 +1366,17 @@ class SpendLogCleanup:
await self._run(prisma_client, age_rules=False, size_rule=True)
async def _run(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> None:
+ """
+ One pass at a time per pod. The age job and the size job fire from separate schedules, and the pod
+ lock only exists with Redis, so this is what keeps them apart in a pod without it. The later one waits
+ rather than skips, because the other job applies a different rule and would not do its work for it.
+ """
+ if _PASS_LOCK.locked():
+ verbose_proxy_logger.info("Another cleanup pass is running in this pod; waiting for it to finish")
+ async with _PASS_LOCK:
+ await self._run_passes(prisma_client, age_rules, size_rule)
+
+ async def _run_passes(self, prisma_client: PrismaClient, age_rules: bool, size_rule: bool) -> None:
lock_acquired = False
run_started_at: Final = time.monotonic()
progress: Final = _RunProgress()
diff --git a/tests/unit/proxy/test_spend_log_cleanup.py b/tests/unit/proxy/test_spend_log_cleanup.py
index 8c97f829025..a0f222ed3e0 100644
--- a/tests/unit/proxy/test_spend_log_cleanup.py
+++ b/tests/unit/proxy/test_spend_log_cleanup.py
@@ -20,13 +20,13 @@ from litellm.constants import (
SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS,
)
from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import (
+ SPEND_LOG_CLEANUP_BOUND_SETTINGS,
SizeCleanupState,
+ SpendLogCleanup,
SpendLogStorage,
StorageProbe,
- advance_size_state,
- SPEND_LOG_CLEANUP_BOUND_SETTINGS,
- SpendLogCleanup,
TableCleanupResult,
+ advance_size_state,
)
from litellm.proxy.db.db_transaction_queue.spend_log_cleanup_metrics import (
SpendLogCleanupMetrics,
@@ -1539,8 +1539,8 @@ async def test_progress_reported_for_a_cancelled_run_is_that_run_only(monkeypatc
@pytest.mark.asyncio
-async def test_progress_reported_by_an_overlapping_run_is_its_own(monkeypatch):
- """With APSCHEDULER_MAX_INSTANCES above one, two runs share the cleaner but not their progress"""
+async def test_a_size_pass_that_fires_while_an_age_pass_is_running_in_the_same_pod_waits_for_it(monkeypatch):
+ """Without Redis there is no pod lock, and the two jobs are scheduled on two SpendLogCleanup instances"""
import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module
mock_logger = MagicMock()
@@ -1557,22 +1557,31 @@ async def test_progress_reported_by_an_overlapping_run_is_its_own(monkeypatch):
slow_client = MagicMock()
_wire_tx(slow_client.db)
slow_client.db.execute_raw = _slow_execute_raw
- fast_client = MagicMock()
- _wire_tx(fast_client.db)
- fast_client.db.execute_raw = AsyncMock(side_effect=[150, 150, 0, 0])
+ size_db = MagicMock()
+ _wire_tx(size_db)
+ size_db.execute_raw = AsyncMock(return_value=1000)
+ _wire_size_probes(size_db, [_storage(allocated=10 * 1024**3, live=1000, dead=0)], cutoff=datetime.now(timezone.utc))
+ size_client = MagicMock()
+ size_client.db = size_db
- cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
- cleaner.pod_lock_manager = None
+ age_cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"})
+ age_cleaner.pod_lock_manager = None
+ size_cleaner = _size_cleaner({})
- slow_run = asyncio.ensure_future(cleaner.cleanup_old_spend_logs(slow_client))
+ slow_run = asyncio.ensure_future(age_cleaner.cleanup_old_spend_logs(slow_client))
await asyncio.wait_for(first_batch_done.wait(), timeout=5)
- await cleaner.cleanup_old_spend_logs(fast_client)
+ size_run = asyncio.ensure_future(size_cleaner.cleanup_spend_logs_by_size(size_client))
+ await asyncio.sleep(0.05)
+ assert size_db.deletes.await_count == 0 and not size_run.done(), "the size pass ran while the age pass held the pod"
+
second_run_done.set()
await asyncio.sleep(0)
slow_run.cancel()
with pytest.raises(asyncio.CancelledError):
await slow_run
+ await asyncio.wait_for(size_run, timeout=5)
+ assert size_db.deletes.await_count > 0, "the size pass never ran once the age pass let go"
(error_call,) = mock_logger.error.call_args_list
rendered = error_call[0][0] % error_call[0][1:]
assert "(rows_deleted=100, batches=1)" in rendered
From 81d8c9af38b960ab183b2bd8cea38a2ac81e6060 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 18:46:17 +0000
Subject: [PATCH 018/203] fix(decisions): bill unpriced returned TypeSafe
versions at the requested model and name a missing answers field (#47508)
* test: bill each completed request once and price unpriced returned decision models at the requested model
Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
* fix(logging,decisions): bill each request once and price returned decision models only when they carry rates
Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
* fix(decisions): leave the double-billing fix to #47510 and keep the decisions pricing and missing-answers fixes
* fix(decisions): name malformed answer entries and pick the Arize cache-hit span by attribute
* test(decisions): cover a returned-model row with metadata but no rates
* fix(decisions): keep upstream reply text out of the parse error so it stays a 500
* fix(decisions): answer every malformed upstream reply with a 500, whatever text it carries
A reply that fails the decisions response schema now raises
DecisionsResponseValidationError, which the decisions call maps straight to
InternalServerError instead of passing it through exception_type. Its text
matching read an answer key like rate_limit, or a value like "Request timed
out", as a 429 or a 408, so the router cooled down a deployment for a reply
that fails the same way on every retry.
The message also drops the empty location for a reply that is not an
object, the Arize family test loses the cache_hit entries main already
added, and the Bedrock Mantle count-tokens wire test calls
_metadata_deployment, since main renamed the _deployment helper under it.
---------
Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
---
litellm/cost_calculator.py | 8 ++-
litellm/decisions/call.py | 41 +++++++-----
.../llms/base_llm/decisions/transformation.py | 43 ++++++++++--
...test_arize_otel_v2_openinference_family.py | 1 +
.../test_arize_otel_v2_openinference_spans.py | 18 ++++-
tests/unit/decisions/test_main.py | 67 +++++++++++++++++--
.../test_base_decisions_transformation.py | 22 ++++++
7 files changed, 167 insertions(+), 33 deletions(-)
diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index e8ca60e00af..51f6a65307d 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -906,11 +906,15 @@ def _bedrock_pricing_info(
return model_info_with_pricing_overrides(profile, registration[1])
-def _declares_rates(model_info: ModelInfo | None) -> bool:
- entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None
+def cost_map_key_prices_anything(key: str) -> bool:
+ entry: Final = _raw_cost_map_entry(key)
return entry is not None and _cost_map_entry_prices_anything(entry)
+def _declares_rates(model_info: ModelInfo | None) -> bool:
+ return model_info is not None and cost_map_key_prices_anything(model_info["key"])
+
+
def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | None) -> str | None:
"""The provider to price an explicit base_model under, when the deployment's own provider has
no rates for it (e.g. chatgpt/ subscription rows) but the base model's own row does"""
diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py
index b94b3380ad5..abbeccc10b3 100644
--- a/litellm/decisions/call.py
+++ b/litellm/decisions/call.py
@@ -11,6 +11,7 @@ from pydantic_core import to_jsonable_python
from typing_extensions import assert_never
import litellm
+from litellm.cost_calculator import cost_map_key_prices_anything
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -21,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
BaseDecisionsConfig,
+ DecisionsResponseValidationError,
image_media_type,
image_param,
input_images,
@@ -687,6 +689,13 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]:
return body
+def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str:
+ returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
+ if returned is not None and cost_map_key_prices_anything(returned):
+ return returned
+ return f"{call.custom_llm_provider}/{call.model}"
+
+
def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
formatted: Final = _formatted_response(response, call)
provider_cost: Final = call.provider_config.provider_reported_cost(response)
@@ -694,10 +703,7 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
- "provider_response_model": (
- get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
- or f"{call.custom_llm_provider}/{call.model}"
- ),
+ "provider_response_model": _pricing_model(response, call),
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
}
)
@@ -727,18 +733,21 @@ def _formatted_response(
def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception:
- if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
- provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
- return litellm.APIConnectionError(
- message=f"{provider_label} - {error.message}",
- llm_provider=call.custom_llm_provider,
- model=f"{call.custom_llm_provider}/{call.model}",
- )
- return litellm.exception_type(
- model=f"{call.custom_llm_provider}/{call.model}",
- custom_llm_provider=call.custom_llm_provider,
- original_exception=error,
- )
+ provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
+ model: Final = f"{call.custom_llm_provider}/{call.model}"
+ match error:
+ case DecisionsResponseValidationError():
+ return litellm.InternalServerError(
+ message=f"{provider_label} - {error.message}", llm_provider=call.custom_llm_provider, model=model
+ )
+ case BaseLLMException(status_code_is_synthesized=True):
+ return litellm.APIConnectionError(
+ message=f"{provider_label} - {error.message}", llm_provider=call.custom_llm_provider, model=model
+ )
+ case _:
+ return litellm.exception_type(
+ model=model, custom_llm_provider=call.custom_llm_provider, original_exception=error
+ )
async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py
index 3af023df378..d74257a4a9d 100644
--- a/litellm/llms/base_llm/decisions/transformation.py
+++ b/litellm/llms/base_llm/decisions/transformation.py
@@ -8,6 +8,7 @@ from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
+from pydantic_core import ErrorDetails
from typing_extensions import assert_never
import litellm
@@ -394,7 +395,9 @@ _MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, ob
_LABELS: Final[TypeAdapter[tuple[str, ...]]] = TypeAdapter(tuple[str, ...])
-def _normalize_answer(value: object) -> Mapping[str, object]:
+def _normalize_answer(value: object) -> object:
+ if not isinstance(value, Mapping):
+ return value
answer: Final = _MAPPING.validate_python(value)
legend: Final = answer.get("legend")
if answer.get("type") != "score" or not isinstance(legend, list):
@@ -403,11 +406,36 @@ def _normalize_answer(value: object) -> Mapping[str, object]:
return {**answer, "legend": {str(index): label for index, label in enumerate(labels)}}
-def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
+def _normalize_systemone_response(payload: object) -> object:
+ if not isinstance(payload, Mapping):
+ return payload
response: Final = _MAPPING.validate_python(payload)
- answers: Final = _MAPPING.validate_python(response.get("answers"))
- normalized: Final = {**response, "answers": {name: _normalize_answer(answer) for name, answer in answers.items()}}
- return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(normalized), request)
+ answers: Final = response.get("answers")
+ if not isinstance(answers, Mapping):
+ return response
+ return {
+ **response,
+ "answers": {name: _normalize_answer(answer) for name, answer in _MAPPING.validate_python(answers).items()},
+ }
+
+
+def _validation_detail(detail: ErrorDetails) -> str:
+ location: Final = ".".join(str(part) for part in detail["loc"])
+ return f"{location}: {detail['msg']}" if location else detail["msg"]
+
+
+def _validation_summary(error: ValidationError) -> str:
+ return "; ".join(_validation_detail(detail) for detail in error.errors(include_url=False, include_input=False))
+
+
+class DecisionsResponseValidationError(BaseLLMException):
+ pass
+
+
+def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
+ return systemone_response_to_ir(
+ _SYSTEMONE_RESPONSE_ADAPTER.validate_python(_normalize_systemone_response(payload)), request
+ )
def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None:
@@ -627,9 +655,10 @@ class BaseDecisionsConfig(ABC):
try:
return self.parse_response(payload, request)
except ValidationError as error:
- raise BaseLLMException(
+ raise DecisionsResponseValidationError(
status_code=500,
- message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: {error}",
+ message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: "
+ f"{error.title} {_validation_summary(error)}",
) from error
def get_error_class(
diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_family.py b/tests/integration/observability/test_arize_otel_v2_openinference_family.py
index 0b0c077cbee..f638a7360be 100644
--- a/tests/integration/observability/test_arize_otel_v2_openinference_family.py
+++ b/tests/integration/observability/test_arize_otel_v2_openinference_family.py
@@ -299,6 +299,7 @@ def test_arize_otel_v2_b3_non_openinference_callback_family(callback: str, gatew
assert _json_object(response.content) == _chat_caller_response(_chat_response(marker), rig.model), response.text
attributes: Final = _matching_genai_marker_span(rig.destination, marker)
assert frozenset(attributes) == _B3_ATTRIBUTE_KEYS[callback], attributes
+ assert attributes["litellm.cache_hit"] == "False", attributes
assert "metadata" not in attributes, attributes
assert not any(".tool_calls." in key for key in attributes), attributes
baggage: Final = tuple(
diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_spans.py b/tests/integration/observability/test_arize_otel_v2_openinference_spans.py
index ab79e81289e..6a7d6d99a84 100644
--- a/tests/integration/observability/test_arize_otel_v2_openinference_spans.py
+++ b/tests/integration/observability/test_arize_otel_v2_openinference_spans.py
@@ -921,7 +921,16 @@ def test_arize_otel_v2_a_cache(surface: str, gateway: Gateway, tmp_path: Path) -
)
assert len(sentinel_spans) == 1, spans
_assert_tool_span_for_marker(sentinel_spans[0], sentinel)
- assert not any(attributes.get("litellm.metadata.trace_marker") == marker for attributes in spans), spans
+ cache_hit_spans: Final = tuple(
+ attributes
+ for attributes in spans
+ if (
+ attributes.get("litellm.metadata.trace_marker") == marker
+ and attributes.get("litellm.cache_hit") == "True"
+ )
+ )
+ assert len(cache_hit_spans) == 1, spans
+ assert cache_hit_spans[0]["litellm.cost.total"] == "0.0", cache_hit_spans
def test_arize_otel_v2_cache_hit_exports_llm_span(gateway: Gateway, tmp_path: Path) -> None:
@@ -960,4 +969,9 @@ def test_arize_otel_v2_cache_hit_exports_llm_span(gateway: Gateway, tmp_path: Pa
attributes for attributes in spans if attributes.get("litellm.metadata.trace_marker") == marker
)
assert len(marker_spans) == 2, spans
- _assert_tool_span_for_marker(marker_spans[1], marker)
+ cache_hit_spans: Final = tuple(
+ attributes for attributes in marker_spans if attributes.get("litellm.cache_hit") == "True"
+ )
+ assert len(cache_hit_spans) == 1, marker_spans
+ _assert_tool_span_for_marker(cache_hit_spans[0], marker)
+ assert cache_hit_spans[0]["litellm.cost.total"] == "0.0", marker_spans
diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py
index 20af774278f..84e7a970195 100644
--- a/tests/unit/decisions/test_main.py
+++ b/tests/unit/decisions/test_main.py
@@ -612,13 +612,35 @@ async def test_oss_classifier_rejects_unknown_models_and_ambiguous_bases_before_
assert len(respx_mock.calls) == 0
-@pytest.mark.asyncio
-async def test_upstream_reply_without_answers_is_a_server_error(respx_mock: respx.MockRouter) -> None:
- respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(
- json={"model": "pplx-decider-v1-27b", "usage": {"input_tokens": 10, "output_tokens": 0}}
- )
+_PPLX_USAGE: Final = {"model": "pplx-decider-v1-27b", "usage": {"input_tokens": 10, "output_tokens": 0}}
- with pytest.raises(litellm.InternalServerError, match="unexpected response"):
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("reply", "detail"),
+ (
+ pytest.param(_PPLX_USAGE, "DecisionsResponse answers: Field required", id="usage"),
+ pytest.param({"error": "Rate limit reached"}, "DecisionsResponse answers: Field required", id="rate_limit_text"),
+ pytest.param({"error": "Request timed out"}, "DecisionsResponse answers: Field required", id="timeout_text"),
+ pytest.param(
+ {**_PPLX_USAGE, "answers": {"rate_limit": None}},
+ "DecisionsResponse answers.rate_limit: Input should be a valid dictionary",
+ id="null_answer_named_rate_limit",
+ ),
+ pytest.param(
+ {**_PPLX_USAGE, "answers": {"is_defect": {"type": "Request timed out"}}},
+ "DecisionsResponse answers.is_defect",
+ id="answer_type_timeout_text",
+ ),
+ pytest.param(["Rate limit reached"], "DecisionsResponse Input should be a valid dictionary", id="not_an_object"),
+ ),
+)
+async def test_a_malformed_upstream_reply_is_a_server_error_whatever_its_text(
+ reply: object, detail: str, respx_mock: respx.MockRouter
+) -> None:
+ respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=reply)
+
+ with pytest.raises(litellm.InternalServerError, match=re.escape(f"unexpected response: {detail}")):
await litellm.asystemone(
model="perplexity/pplx-decider-v1-27b",
state="review",
@@ -3690,6 +3712,39 @@ async def test_decision_hosts_preserve_drex_model_routing_credentials_and_return
}
+@pytest.mark.parametrize(
+ "unpriced_row",
+ (
+ pytest.param({}, id="empty"),
+ pytest.param(
+ {"litellm_provider": "typesafe", "mode": "evaluation", "input_cost_per_token": None},
+ id="metadata_and_null_rate",
+ ),
+ ),
+)
+async def test_a_returned_model_registered_without_prices_is_billed_at_the_requested_model(
+ unpriced_row: dict[str, object], monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
+) -> None:
+ monkeypatch.setenv("TYPESAFE_API_KEY", "typesafe-test-key")
+ monkeypatch.setitem(
+ litellm.model_cost,
+ "typesafe/jev-test-alias",
+ {"litellm_provider": "typesafe", "mode": "evaluation", "input_cost_per_token": 0.001, "output_cost_per_token": 0},
+ )
+ monkeypatch.setitem(litellm.model_cost, "typesafe/jev-test-unpriced", unpriced_row)
+ respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json={**_RESPONSE, "model": "jev-test-unpriced"})
+
+ response: Final = await litellm.asystemone(
+ model="typesafe/jev-test-alias",
+ state="damaged order",
+ questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
+ )
+
+ assert litellm.completion_cost(completion_response=response, call_type="asystemone") == pytest.approx(
+ _INPUT_TOKENS * 0.001
+ )
+
+
@pytest.mark.parametrize("base_env", ("NACE_API_BASE", "NACE_BASE_URL"))
def test_nace_sync_call_uses_environment_base_and_explicit_credentials(
base_env: str,
diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py
index 924b192a00e..3aebf56ee73 100644
--- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py
+++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py
@@ -197,6 +197,28 @@ def test_missing_or_mismatched_systemone_answers_are_refusals_left_out_of_the_re
assert tuple(ir_to_systemone_response(parsed, ir).answers) == ("damaged",)
+@pytest.mark.parametrize(
+ ("payload", "field"),
+ [
+ ({"model": "jev-latest", "usage": {"input_tokens": 10, "output_tokens": 1}}, ("answers",)),
+ (
+ {"model": "jev-latest", "answers": {"damaged": None}, "usage": {"input_tokens": 10, "output_tokens": 1}},
+ ("answers", "damaged"),
+ ),
+ ],
+ ids=["missing_answers", "null_answer"],
+)
+def test_a_malformed_systemone_response_names_the_field_at_fault(
+ payload: Mapping[str, object], field: tuple[str, ...]
+) -> None:
+ ir: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python(_SYSTEMONE_REQUEST))
+
+ with pytest.raises(ValidationError) as error:
+ parse_systemone_response(payload, ir)
+
+ assert [item["loc"] for item in error.value.errors()] == [field]
+
+
def test_each_media_kind_is_gated_by_its_own_support_flag() -> None:
audio: Final = ["data:audio/wav;base64,YQ=="]
videos: Final = ["data:video/mp4;base64,YQ=="]
From 7655c2c839108b47cbda5df82dabef217198d53e Mon Sep 17 00:00:00 2001
From: Abhishek Acharya <118182738+abhishekacharya200@users.noreply.github.com>
Date: Wed, 21 Oct 2026 00:43:43 +0545
Subject: [PATCH 019/203] fix(bedrock): return a 400 for malformed batch JSONL
lines (#45992)
* fix(bedrock): return a 400 for malformed batch JSONL lines
A malformed line in a Bedrock batch input file escaped as a JSONDecodeError
from the upload and file content conversions, which the proxy reported as a
500. Parse those files line by line and raise a 400 that names the bad line.
Object name inference keeps its fallback for files that are not JSONL.
Fixes #45060
* test(bedrock): cover blank lines in malformed batch JSONL
* fix(bedrock): reject a malformed batch line whatever the upload's MIME type
The upload only converts, and so re-checks, batch files whose MIME type is on the
allowlist. Falling back to the generic object name let a malformed file sent as
application/x-jsonl skip both checks and land in S3 unchanged, so the object-name
step now runs the same strict per-line parse and raises the 400.
* fix(bedrock): reject every malformed batch input line with a 400 naming the line
Non-object lines, a missing or non-object body, a non-string body.model,
non-UTF-8 bytes, nesting too deep to parse, and rows the Bedrock mapping
cannot convert escaped as AttributeError, TypeError, UnicodeDecodeError or
RecursionError and surfaced as 500s. They now raise a 400 BedrockError in
the same "Batch input file line N ..." wording the proxy pre-check and the
Vertex adapter use. Object naming reads only the first line and never
raises, and a UTF-8 byte order mark is accepted.
* docs(bedrock): point the batch file name comment at the helper that builds it
* test(bedrock): mark new batch test locals Final and give the line builders explicit params
* test(bedrock): use single quotes inside the batch-line f-strings so the file parses on Python 3.10 and 3.11
---------
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Co-authored-by: ryan-crabbe-berri
---
.../llms/bedrock/batches/transformation.py | 2 +-
litellm/llms/bedrock/files/transformation.py | 254 +++++++++++-------
.../test_bedrock_files_transformation.py | 218 ++++++++++++++-
3 files changed, 364 insertions(+), 110 deletions(-)
diff --git a/litellm/llms/bedrock/batches/transformation.py b/litellm/llms/bedrock/batches/transformation.py
index 3ad8049ec26..50fb911709a 100644
--- a/litellm/llms/bedrock/batches/transformation.py
+++ b/litellm/llms/bedrock/batches/transformation.py
@@ -42,7 +42,7 @@ if TYPE_CHECKING:
# Bedrock batch input files are uploaded as
# s3://bucket/litellm-bedrock-files-{model, ":" -> "-"}-{uuid4}.jsonl (see
-# BedrockFilesTransformation._get_s3_object_name). A uuid4 is always 36 hex/dash
+# _batch_s3_object_name in litellm/llms/bedrock/files/transformation.py). A uuid4 is always 36 hex/dash
# characters, so it can be stripped off the end unambiguously even though the
# model name itself may contain dashes.
_S3_BATCH_FILE_UUID_SUFFIX_PATTERN: Final = re.compile(
diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py
index ce2c8f15d66..cb10c2384cd 100644
--- a/litellm/llms/bedrock/files/transformation.py
+++ b/litellm/llms/bedrock/files/transformation.py
@@ -4,7 +4,7 @@ import os
import posixpath
import time
import xml.etree.ElementTree as ET
-from collections.abc import Iterable, Mapping, MutableMapping, Sequence
+from collections.abc import Iterable, Iterator, Mapping, MutableMapping, Sequence
from contextlib import suppress
from dataclasses import dataclass
from datetime import datetime
@@ -23,6 +23,7 @@ from typing_extensions import ReadOnly
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.constants import BEDROCK_INVOKE_PROVIDERS_LITERAL
+from litellm.exceptions import BadRequestError
from litellm.files.utils import FilesAPIUtils
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.cloud_storage_security import (
@@ -324,7 +325,10 @@ _REJECTED_FILE_ID_REQUEST_URL: Final = "https://litellm.ai"
def _rejected_file_id(reason: ValueError) -> BedrockError:
- message: Final = str(reason)
+ return _client_error(str(reason))
+
+
+def _client_error(message: str) -> BedrockError:
return BedrockError(
status_code=400,
message=message,
@@ -336,6 +340,92 @@ def _rejected_file_id(reason: ValueError) -> BedrockError:
)
+@dataclass(frozen=True, slots=True)
+class _BatchInputLine:
+ line_number: int
+ record: _OpenAIBatchRecord
+
+
+_BATCH_RECORD_CONVERSION_ERRORS: Final = (AttributeError, LookupError, TypeError, ValueError, BadRequestError)
+
+
+def _batch_line_error(line_number: int, problem: str) -> BedrockError:
+ return _client_error(f"Batch input file line {line_number} {problem}")
+
+
+def _decode_batch_input(content: str | bytes) -> str:
+ if isinstance(content, str):
+ return content.removeprefix("\ufeff")
+ try:
+ return content.decode("utf-8-sig")
+ except UnicodeDecodeError as e:
+ raise _batch_line_error(content.count(b"\n", 0, e.start) + 1, f"is not valid UTF-8: {e.reason}") from e
+
+
+def _load_batch_line(line_number: int, line: str) -> _OpenAIBatchRecord:
+ try:
+ return json.loads(line)
+ except json.JSONDecodeError as e:
+ raise _batch_line_error(line_number, f"is not valid JSON: {e.msg} at column {e.colno}") from e
+ except (ValueError, RecursionError) as e:
+ raise _batch_line_error(line_number, f"could not be parsed: {e}") from e
+
+
+def _json_object(value: object) -> Mapping[str, object] | None:
+ return value if isinstance(value, dict) else None
+
+
+def _batch_json_object(line_number: int, line: str) -> _OpenAIBatchRecord:
+ record: Final = _load_batch_line(line_number, line)
+ if _json_object(record) is None:
+ raise _batch_line_error(line_number, "is not a JSON object")
+ return record
+
+
+def _batch_body_problem(record: Mapping[str, object]) -> str | None:
+ body: Final = _json_object(record.get("body"))
+ if body is None:
+ return "has a body that is missing or not a JSON object"
+ if "model" in body and not isinstance(body["model"], str):
+ return "has a body.model that is not a string"
+ return None
+
+
+def _batch_input_line(line_number: int, line: str) -> _BatchInputLine:
+ record: Final = _batch_json_object(line_number, line)
+ problem: Final = _batch_body_problem(record)
+ if problem is not None:
+ raise _batch_line_error(line_number, problem)
+ return _BatchInputLine(line_number=line_number, record=record)
+
+
+def _non_blank_lines(file_content: str) -> Iterator[tuple[int, str]]:
+ numbered_lines: Final = enumerate(file_content.splitlines(), start=1)
+ return ((line_number, line) for line_number, line in numbered_lines if line.strip())
+
+
+def _parse_batch_jsonl(file_content: str) -> tuple[_BatchInputLine, ...]:
+ return tuple(_batch_input_line(line_number, line) for line_number, line in _non_blank_lines(file_content))
+
+
+def _batch_file_model(file_content: str) -> str | None:
+ records: Final = tuple(
+ _batch_json_object(line_number, line) for line_number, line in _non_blank_lines(file_content)
+ )
+ if not records:
+ return None
+ body: Final = _json_object(records[0].get("body"))
+ model: Final = body.get("model") if body is not None else None
+ return model if isinstance(model, str) else ""
+
+
+def _batch_s3_object_name(model: str) -> str:
+ safe_model: Final = sanitize_cloud_object_component(
+ model.removeprefix("bedrock/").replace(":", "-"), fallback="model"
+ )
+ return f"{BEDROCK_MANAGED_S3_BATCH_PREFIX}{safe_model}-{uuid.uuid4()}.jsonl"
+
+
def _resolve_managed_s3_object(file_id: str, litellm_params: Mapping[str, object]) -> tuple[str, str]:
configured_bucket_names: Final = get_configured_s3_bucket_names(litellm_params)
allow_legacy_cloud_file_ids: Final = should_allow_legacy_cloud_file_ids(litellm_params)
@@ -545,29 +635,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
# File-like objects need to be read
content = file_content.read()
- # Ensure content is string
- if isinstance(content, bytes):
- content = content.decode("utf-8")
-
- return content
-
- def _get_s3_object_name_from_batch_jsonl(
- self,
- openai_jsonl_content: Sequence[_OpenAIBatchRecord],
- ) -> str:
- """
- Gets a unique S3 object name for the Bedrock batch processing job
-
- named as: litellm-bedrock-files/{model}/{uuid}
- """
- _model = openai_jsonl_content[0].get("body", {}).get("model", "")
- # Remove bedrock/ prefix if present
- _model = _model.removeprefix("bedrock/")
-
- safe_model: Final = sanitize_cloud_object_component(_model.replace(":", "-"), fallback="model")
-
- object_name: Final = f"{BEDROCK_MANAGED_S3_BATCH_PREFIX}{safe_model}-{uuid.uuid4()}.jsonl"
- return object_name
+ return _decode_batch_input(content)
def get_object_name(self, extracted_file_data: ExtractedFileData, purpose: str) -> str:
"""
@@ -581,11 +649,9 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
if purpose == "batch":
## 1. If jsonl, check if there's a model name
file_content: Final = self._get_content_from_openai_file(extracted_file_data_content)
-
- # Split into lines and parse each line as JSON
- openai_jsonl_content: Final = [json.loads(line) for line in file_content.splitlines() if line.strip()]
- if len(openai_jsonl_content) > 0:
- return self._get_s3_object_name_from_batch_jsonl(openai_jsonl_content)
+ batch_model: Final = _batch_file_model(file_content)
+ if batch_model is not None:
+ return _batch_s3_object_name(batch_model)
## 2. If not jsonl, store under a server-generated managed object name
filename: Final = extracted_file_data.get("filename")
@@ -1080,42 +1146,60 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
}
}
"""
+ return [
+ self._transform_openai_batch_record(idx, openai_record, target_model)
+ for idx, openai_record in enumerate(openai_jsonl_content)
+ ]
+ def transform_batch_input_lines(
+ self, batch_input_lines: Sequence[_BatchInputLine], target_model: str = ""
+ ) -> tuple[_BedrockBatchRecord, ...]:
+ return tuple(
+ self._transform_batch_input_line(idx, batch_input_line, target_model)
+ for idx, batch_input_line in enumerate(batch_input_lines)
+ )
+
+ def _transform_batch_input_line(
+ self, idx: int, batch_input_line: _BatchInputLine, target_model: str
+ ) -> _BedrockBatchRecord:
+ try:
+ return self._transform_openai_batch_record(idx, batch_input_line.record, target_model)
+ except _BATCH_RECORD_CONVERSION_ERRORS as e:
+ raise _batch_line_error(
+ batch_input_line.line_number, f"could not be converted to a Bedrock batch record: {e}"
+ ) from e
+
+ def _transform_openai_batch_record(
+ self, idx: int, openai_record: _OpenAIBatchRecord, target_model: str
+ ) -> _BedrockBatchRecord:
import litellm
- bedrock_jsonl_content: Final = []
- for idx, _openai_jsonl_content in enumerate(openai_jsonl_content):
- # Extract the request body from OpenAI format
- openai_body = _openai_jsonl_content.get("body", {})
- record_model = openai_body.get("model", "")
- resolved_model = litellm.model_alias_map.get(record_model, record_model)
- model_for_transform, provider = self._resolve_batch_record_model_and_provider(
- record_model=resolved_model, target_model=target_model
+ openai_body: Final = openai_record.get("body", {})
+ record_model: Final = openai_body.get("model", "")
+ resolved_model: Final = litellm.model_alias_map.get(record_model, record_model)
+ model_for_transform, provider = self._resolve_batch_record_model_and_provider(
+ record_model=resolved_model, target_model=target_model
+ )
+
+ # Route to the embedding transformer when the OpenAI batch line
+ # targets /v1/embeddings; every other endpoint shape is normalized
+ # to chat completions first. We branch here (rather than inside
+ # `_map_openai_to_bedrock_params`) so the chat helper keeps its
+ # narrow contract and the embedding helper can evolve independently.
+ record_kind: Final = self._classify_batch_record(openai_record)
+ model_input: Final = (
+ self._map_openai_embedding_to_bedrock_params(openai_request_body=openai_body, model=model_for_transform)
+ if record_kind is BedrockBatchRecordKind.EMBEDDING
+ else self._map_openai_to_bedrock_params(
+ openai_request_body=self._transform_batch_body_to_chat_body(openai_body, record_kind),
+ model=model_for_transform,
+ provider=provider,
)
-
- # Route to the embedding transformer when the OpenAI batch line
- # targets /v1/embeddings; every other endpoint shape is normalized
- # to chat completions first. We branch here (rather than inside
- # `_map_openai_to_bedrock_params`) so the chat helper keeps its
- # narrow contract and the embedding helper can evolve independently.
- record_kind = self._classify_batch_record(_openai_jsonl_content)
- if record_kind is BedrockBatchRecordKind.EMBEDDING:
- model_input = self._map_openai_embedding_to_bedrock_params(
- openai_request_body=openai_body, model=model_for_transform
- )
- else:
- model_input = self._map_openai_to_bedrock_params(
- openai_request_body=self._transform_batch_body_to_chat_body(openai_body, record_kind),
- model=model_for_transform,
- provider=provider,
- )
-
- # Create Bedrock batch record
- record_id = _openai_jsonl_content.get("custom_id", f"CALL{str(idx).zfill(7)}")
- bedrock_record = {"recordId": record_id, "modelInput": model_input}
-
- bedrock_jsonl_content.append(bedrock_record)
- return bedrock_jsonl_content
+ )
+ return _BedrockBatchRecord(
+ recordId=openai_record.get("custom_id", f"CALL{str(idx).zfill(7)}"),
+ modelInput=model_input,
+ )
def transform_openai_jsonl_content_to_bedrock_jsonl_content(
self,
@@ -1151,11 +1235,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
):
## Transform JSONL content to Bedrock format
original_file_content: Final = self._get_content_from_openai_file(extracted_file_data_content)
- openai_jsonl_content = [json.loads(line) for line in original_file_content.splitlines() if line.strip()]
+ batch_input_lines: Final = _parse_batch_jsonl(original_file_content)
litellm_params_model: Final = litellm_params.get("model")
target_model: Final = model or (litellm_params_model if isinstance(litellm_params_model, str) else "")
- bedrock_jsonl_content = self._transform_openai_jsonl_content_to_bedrock_jsonl_content(
- openai_jsonl_content, target_model=target_model
+ bedrock_jsonl_content: Final = self.transform_batch_input_lines(
+ batch_input_lines, target_model=target_model
)
file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content)
elif isinstance(extracted_file_data_content, bytes):
@@ -1662,37 +1746,11 @@ class BedrockJsonlFilesTransformation:
# Read the content of the file
file_content: Final = self._get_content_from_openai_file(openai_file_content)
- # Split into lines and parse each line as JSON
- openai_jsonl_content: Final = [json.loads(line) for line in file_content.splitlines() if line.strip()]
- bedrock_jsonl_content = self._transform_openai_jsonl_content_to_bedrock_jsonl_content(openai_jsonl_content)
+ batch_input_lines: Final = _parse_batch_jsonl(file_content)
+ bedrock_jsonl_content: Final = BedrockFilesConfig().transform_batch_input_lines(batch_input_lines)
bedrock_jsonl_string: Final = "\n".join(json.dumps(item) for item in bedrock_jsonl_content)
- object_name: Final = self._get_s3_object_name(openai_jsonl_content=openai_jsonl_content)
- return bedrock_jsonl_string, object_name
-
- def _transform_openai_jsonl_content_to_bedrock_jsonl_content(
- self, openai_jsonl_content: Sequence[_OpenAIBatchRecord]
- ):
- """
- Delegate to the main BedrockFilesConfig transformation method
- """
- config: Final = BedrockFilesConfig()
- return config.transform_openai_jsonl_content_to_bedrock_jsonl_content(openai_jsonl_content)
-
- def _get_s3_object_name(
- self,
- openai_jsonl_content: Sequence[_OpenAIBatchRecord],
- ) -> str:
- """
- Gets a unique S3 object name for the Bedrock batch processing job
-
- named as: litellm-bedrock-files-{model}-{uuid}
- """
- _model = openai_jsonl_content[0].get("body", {}).get("model", "")
- # Remove bedrock/ prefix if present
- _model = _model.removeprefix("bedrock/")
- safe_model: Final = sanitize_cloud_object_component(_model.replace(":", "-"), fallback="model")
- object_name: Final = f"{BEDROCK_MANAGED_S3_BATCH_PREFIX}{safe_model}-{uuid.uuid4()}.jsonl"
- return object_name
+ first_model: Final = batch_input_lines[0].record.get("body", {}).get("model", "") if batch_input_lines else ""
+ return bedrock_jsonl_string, _batch_s3_object_name(first_model)
def _get_content_from_openai_file(self, openai_file_content: FileTypes) -> str:
"""
@@ -1725,11 +1783,7 @@ class BedrockJsonlFilesTransformation:
# File-like objects need to be read
content = file_content.read()
- # Ensure content is string
- if isinstance(content, bytes):
- content = content.decode("utf-8")
-
- return content
+ return _decode_batch_input(content)
def transform_s3_bucket_response_to_openai_file_object(
self, create_file_data: CreateFileRequest, s3_upload_response: _S3UploadResponse
diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py
index d64a9ee845a..7e05ef500f5 100644
--- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py
+++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py
@@ -22,7 +22,8 @@ from botocore.credentials import Credentials
import litellm
from litellm.constants import DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET
-from litellm.llms.bedrock.files.transformation import BedrockJsonlFilesTransformation
+from litellm.llms.bedrock.common_utils import BedrockError
+from litellm.llms.bedrock.files.transformation import BedrockFilesConfig, BedrockJsonlFilesTransformation
from tests.unit.proxy.conftest import httpx_transport
@@ -36,9 +37,6 @@ class TestBedrockFilesTransformation:
Validates that the transformation correctly converts OpenAI batch completion
format to Bedrock's expected batch format with proper recordId and modelInput structure.
"""
- # Initialize the transformation class
- transformation = BedrockJsonlFilesTransformation()
-
# Load input JSONL file
input_file_path = os.path.join(
os.path.dirname(__file__), "input_batch_completions.jsonl"
@@ -52,8 +50,8 @@ class TestBedrockFilesTransformation:
openai_jsonl_content.append(json.loads(line))
# Transform the content
- bedrock_jsonl_content = transformation._transform_openai_jsonl_content_to_bedrock_jsonl_content(
- openai_jsonl_content=openai_jsonl_content
+ bedrock_jsonl_content: Final = BedrockFilesConfig().transform_openai_jsonl_content_to_bedrock_jsonl_content(
+ openai_jsonl_content
)
# Basic validation
@@ -408,9 +406,9 @@ class TestBedrockFilesTransformation:
def test_batch_object_name_sanitizes_model_path(self):
from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
- config = BedrockFilesConfig()
- object_name = config._get_s3_object_name_from_batch_jsonl(
- [{"body": {"model": "bedrock/../../secret:model"}}]
+ object_name: Final = BedrockFilesConfig().get_object_name(
+ {"content": json.dumps({"body": {"model": "bedrock/../../secret:model"}}).encode(), "filename": "b.jsonl"},
+ purpose="batch",
)
assert object_name.startswith("litellm-bedrock-files-")
@@ -4376,3 +4374,205 @@ async def test_async_create_file(
"s3://litellm-proxy-941277531214/litellm-bedrock-files-"
)
assert file_obj.filename.endswith(".jsonl")
+
+
+_VALID_BATCH_LINE: Final = json.dumps(
+ {
+ "custom_id": "req-1",
+ "method": "POST",
+ "url": "/v1/chat/completions",
+ "body": {
+ "model": "bedrock/amazon.nova-pro-v1:0",
+ "messages": [{"role": "user", "content": "Hello"}],
+ "max_tokens": 10,
+ },
+ }
+)
+_MALFORMED_BATCH_JSONL: Final = f"{_VALID_BATCH_LINE}\n\n{{not json\n".encode()
+
+
+class TestMalformedBatchJsonl:
+ """A malformed line in a batch input file is the client's mistake, so it must come back as a 400 that names the
+ line, not escape as a JSONDecodeError the proxy reports as a 500 (#45060)."""
+
+ def test_create_file_request_rejects_a_malformed_line_with_a_400(self):
+ from litellm.llms.bedrock.common_utils import BedrockError
+ from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
+
+ with pytest.raises(BedrockError) as exc_info:
+ BedrockFilesConfig().transform_create_file_request(
+ model="amazon.nova-pro-v1:0",
+ create_file_data={
+ "file": ("batch.jsonl", _MALFORMED_BATCH_JSONL, "application/jsonl"),
+ "purpose": "batch",
+ },
+ optional_params={
+ "aws_access_key_id": "test-key-id",
+ "aws_secret_access_key": "test-secret",
+ "aws_region_name": "us-west-2",
+ },
+ litellm_params={"s3_bucket_name": "litellm-batch-bucket"},
+ )
+ assert exc_info.value.status_code == 400
+ assert "line 3" in exc_info.value.message
+
+ def test_file_content_conversion_rejects_a_malformed_line_with_a_400(self):
+ from litellm.llms.bedrock.common_utils import BedrockError
+
+ with pytest.raises(BedrockError) as exc_info:
+ BedrockJsonlFilesTransformation().transform_openai_file_content_to_bedrock_file_content(
+ _MALFORMED_BATCH_JSONL
+ )
+ assert exc_info.value.status_code == 400
+ assert "line 3" in exc_info.value.message
+
+ @pytest.mark.parametrize("content_type", ["application/jsonl", "application/x-jsonl"])
+ @pytest.mark.parametrize(
+ "file_content",
+ [_MALFORMED_BATCH_JSONL, f"{_VALID_BATCH_LINE}\n\n[1, 2]\n".encode()],
+ ids=["invalid-json", "array-line"],
+ )
+ def test_create_file_rejects_a_malformed_line_before_uploading(self, content_type: str, file_content: bytes):
+ """application/x-jsonl is not on the batch MIME allowlist, so the upload skips conversion for it; the line
+ check must still run so the broken file is rejected instead of landing in S3 unchanged."""
+ from litellm.llms.bedrock.common_utils import BedrockError
+
+ with respx.mock as router:
+ any_request: Final = router.route()
+ with pytest.raises(BedrockError) as exc_info:
+ litellm.create_file(
+ file=("batch.jsonl", file_content, content_type),
+ purpose="batch",
+ custom_llm_provider="bedrock",
+ model="bedrock/amazon.nova-pro-v1:0",
+ s3_bucket_name="litellm-batch-bucket",
+ aws_region_name="us-west-2",
+ aws_access_key_id="test-key-id",
+ aws_secret_access_key="test-secret",
+ )
+ assert exc_info.value.status_code == 400
+ assert "line 3" in exc_info.value.message
+ assert not any_request.called
+
+
+_VALID_BATCH_RECORD: Final = json.loads(_VALID_BATCH_LINE)
+
+
+def _batch_line(body: object) -> str:
+ return json.dumps({**_VALID_BATCH_RECORD, "body": body})
+
+
+def _batch_line_with_body(field: str, value: object) -> str:
+ return _batch_line({**_VALID_BATCH_RECORD["body"], field: value})
+
+
+@pytest.mark.usefixtures(httpx_transport.__name__)
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("file_content", "expected_message"),
+ [
+ pytest.param(
+ f"{_VALID_BATCH_LINE}\n[1, 2]\n".encode(), "Batch input file line 2 is not a JSON object", id="array-line"
+ ),
+ pytest.param(
+ f"[1, 2]\n{_VALID_BATCH_LINE}\n".encode(), "Batch input file line 1 is not a JSON object", id="array-first"
+ ),
+ pytest.param(
+ f"{_VALID_BATCH_LINE}\n{_batch_line(None)}\n".encode(),
+ "Batch input file line 2 has a body that is missing or not a JSON object",
+ id="null-body",
+ ),
+ pytest.param(
+ f"{_batch_line('hello')}\n".encode(),
+ "Batch input file line 1 has a body that is missing or not a JSON object",
+ id="string-body",
+ ),
+ pytest.param(
+ f"{_batch_line_with_body('model', 123)}\n".encode(),
+ "Batch input file line 1 has a body.model that is not a string",
+ id="int-model",
+ ),
+ pytest.param(
+ f"{_batch_line_with_body('model', None)}\n".encode(),
+ "Batch input file line 1 has a body.model that is not a string",
+ id="null-model",
+ ),
+ pytest.param(
+ f"{_VALID_BATCH_LINE}\n{_batch_line_with_body('messages', 'hi')}\n".encode(),
+ "Batch input file line 2 could not be converted to a Bedrock batch record",
+ id="string-messages",
+ ),
+ pytest.param(
+ f"{_VALID_BATCH_LINE}\n{'[' * 1_000_000}{']' * 1_000_000}\n".encode(),
+ "Batch input file line 2 could not be parsed",
+ id="deep-nesting",
+ ),
+ pytest.param(
+ f"{_VALID_BATCH_LINE}\n".encode() + b'{"custom_id": "\xff"}\n',
+ "Batch input file line 2 is not valid UTF-8",
+ id="non-utf8",
+ ),
+ ],
+)
+async def test_create_file_rejects_a_malformed_batch_line_with_a_400_before_uploading(
+ respx_mock: respx.MockRouter, file_content: bytes, expected_message: str
+) -> None:
+ put_route: Final = respx_mock.put(url__regex=r"https://s3\..+").respond(status_code=200)
+
+ with pytest.raises(BedrockError) as exc_info:
+ await litellm.acreate_file(
+ file=("batch.jsonl", file_content, "application/jsonl"),
+ purpose="batch",
+ custom_llm_provider="bedrock",
+ s3_bucket_name="litellm-batch-bucket",
+ aws_region_name="us-west-2",
+ aws_access_key_id="test-access-key",
+ aws_secret_access_key="test-secret-key",
+ )
+
+ assert exc_info.value.status_code == 400
+ assert exc_info.value.message.startswith(expected_message)
+ assert not put_route.called
+
+
+@pytest.mark.usefixtures(httpx_transport.__name__)
+@pytest.mark.asyncio
+async def test_create_file_accepts_a_batch_file_with_a_utf8_byte_order_mark(respx_mock: respx.MockRouter) -> None:
+ put_route: Final = respx_mock.put(url__regex=r"https://s3\..+").respond(status_code=200)
+
+ await litellm.acreate_file(
+ file=("batch.jsonl", f"\ufeff{_VALID_BATCH_LINE}\n".encode(), "application/jsonl"),
+ purpose="batch",
+ custom_llm_provider="bedrock",
+ s3_bucket_name="litellm-batch-bucket",
+ aws_region_name="us-west-2",
+ aws_access_key_id="test-access-key",
+ aws_secret_access_key="test-secret-key",
+ )
+
+ uploaded_record: Final = json.loads(put_route.calls.last.request.content)
+ assert uploaded_record["recordId"] == _VALID_BATCH_RECORD["custom_id"]
+ assert "/litellm-bedrock-files-amazon.nova-pro-v1-0-" in str(put_route.calls.last.request.url)
+
+
+@pytest.mark.usefixtures(httpx_transport.__name__)
+@pytest.mark.asyncio
+async def test_create_file_uploads_a_bedrock_format_batch_file_unchanged_when_its_mime_type_skips_conversion(
+ respx_mock: respx.MockRouter,
+) -> None:
+ put_route: Final = respx_mock.put(url__regex=r"https://s3\..+").respond(status_code=200)
+ bedrock_line: Final = json.dumps(
+ {"recordId": "req-1", "modelInput": {"messages": [{"role": "user", "content": [{"text": "Hello"}]}]}}
+ )
+
+ await litellm.acreate_file(
+ file=("batch.jsonl", f"{bedrock_line}\n".encode(), "application/x-jsonl"),
+ purpose="batch",
+ custom_llm_provider="bedrock",
+ s3_bucket_name="litellm-batch-bucket",
+ aws_region_name="us-west-2",
+ aws_access_key_id="test-access-key",
+ aws_secret_access_key="test-secret-key",
+ )
+
+ assert put_route.calls.last.request.content == f"{bedrock_line}\n".encode()
From 04eaa6b4a86d0513bc4b87ca932315379a584fe3 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 19:02:18 +0000
Subject: [PATCH 020/203] refactor(types): replace Any with proven types in 15
files (#47509)
* refactor(types): replace Any with proven types in 9 shared files
* refactor(types): replace Any with proven types in the remaining sweep files
* refactor(types): keep unproven integrations on the merge base and pin malformed-body handling
* refactor(types): leave five request paths on the merge base so non-object bodies keep their 400
* refactor(types): drop a docstring that repeats the helper name
* refactor(types): leave six files with no live malformed-payload proof on the merge base
* refactor(types): leave Anthropic pass-through SSE parsing on the merge base
* refactor(types): leave Responses output walking, the retry ceiling and concatenated tool JSON on the merge base
* fix(types): keep main's parsing in the budget window reset, utils loaders and Langfuse attributes
The new strict checks rejected inputs main skips or keeps, such as a JSONB
string in budget_limits, so these three files go back to main's version
* refactor(types): return the request tag parse failure as a value
---
.../litellm_completion_bridge/handler.py | 2 +-
.../harness/handlers/deepagents_handler.py | 2 +-
litellm/interactions/streaming_iterator.py | 7 ++-
litellm/llms/anthropic/common_utils.py | 55 ++++++++++---------
.../responses_adapters/transformation.py | 43 +++++++--------
.../guardrail_translation/base_translation.py | 2 +-
.../base_managed_resource.py | 4 +-
litellm/llms/chatgpt/chat/transformation.py | 2 +-
litellm/proxy/db/db_spend_update_writer.py | 35 ++++++++----
.../litellm_content_filter/patterns.py | 26 ++++++++-
.../management_endpoints/team_endpoints.py | 6 +-
.../spend_management_endpoints.py | 17 ++----
litellm/proxy/utils.py | 2 +-
litellm/types/proxy/auth/user_api_key_auth.py | 6 +-
.../proxy/management_endpoints/scim_v2.py | 5 +-
.../llms/chatgpt/chat/test_streaming_utils.py | 18 ++++++
16 files changed, 145 insertions(+), 87 deletions(-)
diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py
index 0002a597d6a..6d3b09aed68 100644
--- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py
+++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py
@@ -80,7 +80,7 @@ class A2ACompletionBridgeHandler:
@staticmethod
def _build_completion_params(
params: dict[str, Any],
- litellm_params: Mapping[str, Any],
+ litellm_params: Mapping[str, object],
api_base: str | None,
agent_extra_headers: Mapping[str, str] | None,
*,
diff --git a/litellm/harness/handlers/deepagents_handler.py b/litellm/harness/handlers/deepagents_handler.py
index 102f4a741cf..4a36b1a08d1 100644
--- a/litellm/harness/handlers/deepagents_handler.py
+++ b/litellm/harness/handlers/deepagents_handler.py
@@ -183,7 +183,7 @@ class DeepAgentsHandler(BaseHarnessHandler):
break
resume: dict[str, object] = {} # mutable-ok: Command(resume=) payload, filled per answered approval
for interrupt in state.interrupts:
- decisions: list[dict[str, Any]] = [] # mutable-ok: HITL decisions collected across awaited approvals
+ decisions: list[dict[str, str]] = [] # mutable-ok: HITL decisions collected across awaited approvals
for request in approval_requests(getattr(interrupt, "value", None)):
approval = Approval(
tool=normalized_tool_name(str(request.get("name") or "")),
diff --git a/litellm/interactions/streaming_iterator.py b/litellm/interactions/streaming_iterator.py
index a2468cd894f..f4ad57bf7b4 100644
--- a/litellm/interactions/streaming_iterator.py
+++ b/litellm/interactions/streaming_iterator.py
@@ -11,6 +11,7 @@ from datetime import datetime
from typing import Any, Final
import httpx
+from pydantic import ConfigDict, TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import STREAM_SSE_DONE_STRING
@@ -25,6 +26,10 @@ from litellm.types.interactions import (
)
from litellm.utils import CustomStreamWrapper
+_INTERACTIONS_STREAMING_CHUNK_ADAPTER: Final = TypeAdapter(
+ object, config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
+
class BaseInteractionsAPIStreamingIterator:
"""
@@ -83,7 +88,7 @@ class BaseInteractionsAPIStreamingIterator:
try:
# Parse the JSON chunk
- parsed_chunk: Final = json.loads(stripped_chunk)
+ parsed_chunk: Final = _INTERACTIONS_STREAMING_CHUNK_ADAPTER.validate_python(json.loads(stripped_chunk))
# Format as InteractionsAPIStreamingResponse
if isinstance(parsed_chunk, dict):
diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py
index 0db8f5c33ed..6f08d6b0d61 100644
--- a/litellm/llms/anthropic/common_utils.py
+++ b/litellm/llms/anthropic/common_utils.py
@@ -712,7 +712,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
optional_params: dict,
model: str,
param: str,
- value: Any,
+ value: object,
drop_params: bool,
output_key: str,
) -> None:
@@ -1788,6 +1788,21 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(
data.pop("thinking", None)
+_DROP_ANTHROPIC_MESSAGE: Final = object()
+
+
+def _strip_empty_content_blocks_from_message(message: object) -> object:
+ if not isinstance(message, dict) or not isinstance(message.get("content"), list):
+ return message
+ content: Final = message["content"]
+ filtered: Final = [
+ block for block in content if not _is_empty_text_block(block) and not is_empty_unsigned_thinking_block(block)
+ ]
+ if len(filtered) == len(content):
+ return message
+ return {**message, "content": filtered} if filtered else _DROP_ANTHROPIC_MESSAGE
+
+
def strip_empty_content_blocks_from_anthropic_messages(
messages: Sequence[object],
) -> list[Any]:
@@ -1822,18 +1837,11 @@ def strip_empty_content_blocks_from_anthropic_messages(
The caller's list and its content blocks are never mutated; modified
messages are returned as shallow copies with a fresh content list.
"""
- out: Final[list[Any]] = []
- for m in messages:
- if not isinstance(m, dict) or not isinstance(m.get("content"), list):
- out.append(m)
- continue
- content = m["content"]
- filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_unsigned_thinking_block(b)]
- if len(filtered) == len(content):
- out.append(m)
- elif filtered:
- out.append({**m, "content": filtered})
- return out
+ return [
+ stripped_message
+ for message in messages
+ if (stripped_message := _strip_empty_content_blocks_from_message(message)) is not _DROP_ANTHROPIC_MESSAGE
+ ]
def _is_empty_text_block(block: object) -> bool:
@@ -1921,18 +1929,15 @@ def sanitize_tool_use_ids_in_anthropic_messages(messages: Sequence[object]) -> l
and ``:`` — valid on the upstream provider but rejected by Anthropic when
the session is switched to a native Anthropic deployment.
"""
- out: Final[list[Any]] = []
- for m in messages:
- if not isinstance(m, dict) or not isinstance(m.get("content"), list):
- out.append(m)
- continue
- content = m["content"]
- new_content = [_sanitize_tool_use_id_content_block(b) for b in content]
- if new_content == content:
- out.append(m)
- else:
- out.append({**m, "content": new_content})
- return out
+ return [_sanitize_tool_use_ids_in_message(message) for message in messages]
+
+
+def _sanitize_tool_use_ids_in_message(message: object) -> object:
+ if not isinstance(message, dict) or not isinstance(message.get("content"), list):
+ return message
+ content: Final = message["content"]
+ new_content: Final = [_sanitize_tool_use_id_content_block(block) for block in content]
+ return message if new_content == content else {**message, "content": new_content}
class _ReplayedSearchQuery(LiteLLMBaseModel):
diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py
index 0fafd4abd62..bd0f57c3ed9 100644
--- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py
+++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py
@@ -464,33 +464,32 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
return input_items
+ @staticmethod
+ def _translate_tool_to_responses_api(tool_dict: Mapping[str, object]) -> dict[str, object]:
+ tool_type: Final = tool_dict.get("type", "")
+ tool_name: Final = tool_dict.get("name", "")
+ # web_search tool
+ if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":
+ return {"type": "web_search_preview"}
+ # Responses turns strict mode on when `strict` is omitted, silently rewriting
+ # `required` to every property. Anthropic tools are non-strict unless asked.
+ func_tool: Final[dict[str, object]] = {
+ "type": "function",
+ "name": tool_name,
+ "strict": bool(tool_dict.get("strict")),
+ }
+ if "description" in tool_dict:
+ func_tool["description"] = tool_dict["description"]
+ if "input_schema" in tool_dict:
+ func_tool["parameters"] = json_copy_or_original(tool_dict["input_schema"])
+ return func_tool
+
def translate_tools_to_responses_api(
self,
tools: list[AllAnthropicToolsValues],
) -> list[dict[str, object]]:
"""Convert Anthropic tool definitions to Responses API function tools."""
- result: Final[list[dict[str, object]]] = []
- for tool in tools:
- tool_dict = cast(dict[str, Any], tool)
- tool_type = tool_dict.get("type", "")
- tool_name = tool_dict.get("name", "")
- # web_search tool
- if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":
- result.append({"type": "web_search_preview"})
- continue
- # Responses turns strict mode on when `strict` is omitted, silently rewriting
- # `required` to every property. Anthropic tools are non-strict unless asked.
- func_tool: dict[str, object] = {
- "type": "function",
- "name": tool_name,
- "strict": bool(tool_dict.get("strict")),
- }
- if "description" in tool_dict:
- func_tool["description"] = tool_dict["description"]
- if "input_schema" in tool_dict:
- func_tool["parameters"] = json_copy_or_original(tool_dict["input_schema"])
- result.append(func_tool)
- return result
+ return [self._translate_tool_to_responses_api(tool) for tool in tools]
@staticmethod
def translate_tool_choice_to_responses_api(
diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py
index ee2f6eb9f1a..5fe81a7f1b7 100644
--- a/litellm/llms/base_llm/guardrail_translation/base_translation.py
+++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py
@@ -185,7 +185,7 @@ class BaseTranslation(ABC):
request_data: dict | None = None,
stream_transform_sink: StreamTransformSink | None = None,
deliver_ended_stream_rewrites: bool = False,
- ) -> Any:
+ ) -> object:
"""
Process output streaming response with guardrails.
diff --git a/litellm/llms/base_llm/managed_resources/base_managed_resource.py b/litellm/llms/base_llm/managed_resources/base_managed_resource.py
index bb4cf42fa1e..d3b05716452 100644
--- a/litellm/llms/base_llm/managed_resources/base_managed_resource.py
+++ b/litellm/llms/base_llm/managed_resources/base_managed_resource.py
@@ -28,7 +28,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient as _PrismaClient
from litellm.router import Router as _Router
- Span = _Span | Any
+ Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
Router = _Router
@@ -550,7 +550,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
limit: int | None = None,
after: str | None = None,
additional_filters: Mapping[str, object] | None = None,
- ) -> dict[str, Any]:
+ ) -> dict[str, object]:
"""
List resources created by a user.
diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.py
index 7c985b7dd5e..a1aac7f315b 100644
--- a/litellm/llms/chatgpt/chat/transformation.py
+++ b/litellm/llms/chatgpt/chat/transformation.py
@@ -72,7 +72,7 @@ class ChatGPTConfig(OpenAIConfig):
default_headers: Final = get_chatgpt_default_headers(api_key or "", account_id, session_id)
return {**default_headers, **validated_headers}
- def post_stream_processing(self, stream: Any) -> Any:
+ def post_stream_processing(self, stream: Any) -> ChatGPTToolCallNormalizer:
return ChatGPTToolCallNormalizer(stream)
def map_openai_params(
diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py
index 166a42adfc0..58a801eebe4 100644
--- a/litellm/proxy/db/db_spend_update_writer.py
+++ b/litellm/proxy/db/db_spend_update_writer.py
@@ -13,7 +13,7 @@ import os
import random
import time
import traceback
-from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping, Sequence
+from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence
from contextlib import asynccontextmanager
from contextvars import ContextVar
from datetime import datetime, timedelta, timezone
@@ -21,7 +21,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, overload
from urllib.parse import quote, unquote
-from pydantic import TypeAdapter
+from pydantic import ConfigDict, TypeAdapter
from typing_extensions import LiteralString, ReadOnly, TypedDict
import litellm
@@ -102,6 +102,24 @@ else:
RESPONSES_SESSION_CALL_TYPES: Final = frozenset({CallTypes.responses.value, CallTypes.aresponses.value})
_SPEND_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+_JsonPayload: TypeAlias = Mapping[str, object] | Sequence[object] | str | int | float | bool | None
+_JSON_VALUE_ADAPTER: Final[TypeAdapter[_JsonPayload]] = TypeAdapter(
+ _JsonPayload, config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
+
+
+def _parsed_request_tags(request_tags: object) -> Iterable[object] | TypeError | None:
+ if isinstance(request_tags, list):
+ return request_tags
+ if not isinstance(request_tags, str):
+ return None
+ parsed: Final = _JSON_VALUE_ADAPTER.validate_python(safe_json_loads(request_tags, default=[]))
+ if not parsed:
+ verbose_proxy_logger.debug("Failed to parse request_tags JSON: %s", request_tags)
+ return None
+ if isinstance(parsed, (Sequence, Mapping)):
+ return parsed
+ return TypeError(f"'{type(parsed).__name__}' object is not iterable")
def _org_member_transaction_key(org_id: str, user_id: str) -> str:
@@ -1441,16 +1459,11 @@ class DBSpendUpdateWriter:
return
# Parse tags from JSON string
- tags: Sequence[object] = []
- if isinstance(request_tags, str):
- tags = safe_json_loads(request_tags, default=[])
- if not tags:
- verbose_proxy_logger.debug("Failed to parse request_tags JSON: %s", request_tags)
- return
- elif isinstance(request_tags, list):
- tags = request_tags
- else:
+ tags: Final[Iterable[object] | TypeError | None] = _parsed_request_tags(request_tags)
+ if tags is None:
return
+ if isinstance(tags, TypeError):
+ raise tags
# Update spend for each tag
for tag_name in tags:
diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py
index 9d051eb90d6..da937bd0325 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py
@@ -8,19 +8,39 @@ sensitive information like SSNs, credit cards, API keys, etc.
import json
import os
import re
-from collections.abc import Iterator
+from collections.abc import Iterator, Sequence
from enum import Enum
from re import Pattern
from typing import Any, Final
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import ReadOnly, TypedDict
+
from litellm.proxy.guardrails.content_filter_data import DATA_ROOTS, category_dirs
-def _load_patterns_from_json() -> dict:
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _PatternEntry(TypedDict):
+ name: ReadOnly[str]
+ display_name: ReadOnly[str]
+ pattern: ReadOnly[str]
+ category: ReadOnly[str]
+ description: ReadOnly[str]
+
+
+@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
+class _PatternsFile(TypedDict):
+ patterns: ReadOnly[Sequence[_PatternEntry]]
+
+
+_PATTERNS_FILE: Final = TypeAdapter(_PatternsFile)
+
+
+def _load_patterns_from_json() -> _PatternsFile:
"""Load pattern definitions from patterns.json file"""
json_path: Final = os.path.join(os.path.dirname(__file__), "patterns.json")
with open(json_path, "r") as f:
- return json.load(f)
+ return _PATTERNS_FILE.validate_python(json.load(f))
# Load patterns from JSON
diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py
index 0595d23bd9a..517a05773c8 100644
--- a/litellm/proxy/management_endpoints/team_endpoints.py
+++ b/litellm/proxy/management_endpoints/team_endpoints.py
@@ -1848,7 +1848,7 @@ async def new_team(
members_with_roles = complete_team_data.members_with_roles
complete_team_data.members_with_roles = []
- complete_team_data_dict = complete_team_data.model_dump(exclude_none=True)
+ complete_team_data_dict: dict[str, object] = complete_team_data.model_dump(exclude_none=True)
# Serialize router_settings to JSON (matching key creation pattern)
router_settings_value: Final = data.router_settings
@@ -2563,8 +2563,8 @@ async def update_team(
# Drop server-owned metadata keys from caller input so they can only
# be written by the same code path that creates the underlying rows.
- if isinstance(updated_kv.get("metadata"), dict):
- TeamMemberBudgetHandler.strip_system_managed_metadata_keys(updated_kv["metadata"])
+ if isinstance(metadata_update := updated_kv.get("metadata"), dict):
+ TeamMemberBudgetHandler.strip_system_managed_metadata_keys(metadata_update)
if "metadata" in updated_kv:
stored_metadata: Final[Mapping[str, JsonValue] | None] = (
diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py
index 31732f2dbc9..84278f4619d 100644
--- a/litellm/proxy/spend_tracking/spend_management_endpoints.py
+++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py
@@ -16,7 +16,6 @@ from typing import (
NamedTuple,
Protocol,
TypeAlias,
- TypedDict,
TypeVar,
cast, # noqa: TID251 # custom-logger and cold-storage payloads are untyped JSON
)
@@ -24,7 +23,7 @@ from typing import (
import fastapi
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from pydantic import TypeAdapter
-from typing_extensions import ReadOnly, assert_never
+from typing_extensions import ReadOnly, TypedDict, assert_never
import litellm
from litellm._logging import verbose_proxy_logger
@@ -2721,7 +2720,7 @@ async def ui_view_spend_logs(
end_date_obj = parse_date(end_date)
# Build where conditions
- where_conditions: Final[dict[str, Any]] = {}
+ where_conditions: Final[dict[str, object]] = {}
if start_date_obj is not None and end_date_obj is not None:
where_conditions["startTime"] = {
"gte": start_date_obj.isoformat(), # Already in UTC, no need to add Z
@@ -2783,18 +2782,14 @@ async def ui_view_spend_logs(
if len(metadata_filters) == 1:
where_conditions["metadata"] = metadata_filters[0]
else:
- where_conditions["AND"] = where_conditions.get("AND", []) + [
- {"metadata": filter_cond} for filter_cond in metadata_filters
- ]
+ where_conditions["AND"] = [{"metadata": filter_cond} for filter_cond in metadata_filters]
if end_user is not None:
where_conditions["end_user"] = end_user
if min_spend is not None or max_spend is not None:
- where_conditions["spend"] = {}
- if min_spend is not None:
- where_conditions["spend"]["gte"] = min_spend
- if max_spend is not None:
- where_conditions["spend"]["lte"] = max_spend
+ where_conditions["spend"] = {
+ key: value for key, value in (("gte", min_spend), ("lte", max_spend)) if value is not None
+ }
# A request_id lookup drops the date window, so a non-admin could otherwise
# reach any single row by id; require they own one of the matches, mirroring
# the detail endpoint, and keep the general scoping below so a colliding
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index b01f53c8c6b..2699dab2e29 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -502,7 +502,7 @@ class InternalUsageCache:
key: str,
local_only: bool = False,
**kwargs: object,
- ) -> Any:
+ ) -> object:
return self.dual_cache.get_cache(
key=key,
local_only=local_only,
diff --git a/litellm/types/proxy/auth/user_api_key_auth.py b/litellm/types/proxy/auth/user_api_key_auth.py
index c2f7c487b7b..2c1c75fbe04 100644
--- a/litellm/types/proxy/auth/user_api_key_auth.py
+++ b/litellm/types/proxy/auth/user_api_key_auth.py
@@ -198,14 +198,14 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob
"narrow the agent's access."
),
)
- budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True)
+ budget_reservation: dict[str, object] | None = Field(default=None, exclude=True)
team_budget_snapshot: TeamBudgetSnapshot | None = Field(default=None, exclude=True)
user_budget_snapshot: UserBudgetSnapshot | None = Field(default=None, exclude=True)
org_budget_snapshot: OrgBudgetSnapshot | None = Field(default=None, exclude=True)
matched_model_access_groups: list[str] | None = Field(default=None, exclude=True)
budget_throttle_pct: float | None = Field(default=None, exclude=True)
- user: Any | None = None # Expanded user object when expand=user is used
- created_by_user: Any | None = None # Expanded created_by user when expand=user is used
+ user: object | None = None # Expanded user object when expand=user is used
+ created_by_user: object | None = None # Expanded created_by user when expand=user is used
end_user_object_permission: LiteLLM_ObjectPermissionTable | None = None
# Team object_permission preloaded in auth (e.g. get_team_object) to avoid
# per-request object_permission fetches in downstream checks (vector stores, etc.)
diff --git a/litellm/types/proxy/management_endpoints/scim_v2.py b/litellm/types/proxy/management_endpoints/scim_v2.py
index 9b519f9af4f..fadb309e0cd 100644
--- a/litellm/types/proxy/management_endpoints/scim_v2.py
+++ b/litellm/types/proxy/management_endpoints/scim_v2.py
@@ -21,6 +21,9 @@ SCIM_ROLES_METADATA_KEY: Final = "scim_roles"
SCIM_MANAGED_TEAM_METADATA_KEY: Final = "scim_managed"
SCIM_TEAM_DATA_METADATA_KEY: Final = "scim_data"
+_SCIM_SERIALIZED_FIELDS_ADAPTER: Final = TypeAdapter(
+ dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)
+)
class LiteLLM_UserScimMetadata(LiteLLMBaseModel):
@@ -123,7 +126,7 @@ class SCIMUser(SCIMResource):
@model_serializer(mode="wrap")
def _omit_absent_optional_blocks(self, handler: SerializerFunctionWrapHandler) -> dict[str, object]:
- dumped: Final = handler(self)
+ dumped: Final = _SCIM_SERIALIZED_FIELDS_ADAPTER.validate_python(handler(self))
if self.enterprise_user is None:
dumped.pop(SCIM_ENTERPRISE_USER_SCHEMA, None)
dumped.pop("enterprise_user", None)
diff --git a/tests/unit/llms/chatgpt/chat/test_streaming_utils.py b/tests/unit/llms/chatgpt/chat/test_streaming_utils.py
index 8d95a3954a0..f6c5f768c5d 100644
--- a/tests/unit/llms/chatgpt/chat/test_streaming_utils.py
+++ b/tests/unit/llms/chatgpt/chat/test_streaming_utils.py
@@ -7,9 +7,13 @@ are normalized to match the OpenAI streaming spec:
- Deduplication of "closing" chunks with repeated id/name
"""
+from pathlib import Path
+from typing import Final
+
import pytest
from litellm.llms.chatgpt.chat.streaming_utils import ChatGPTToolCallNormalizer
+from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig
from litellm.types.utils import (
ChatCompletionDeltaToolCall,
Delta,
@@ -201,3 +205,17 @@ class TestChatGPTToolCallNormalizer:
normalizer = ChatGPTToolCallNormalizer(FakeStream())
assert normalizer.custom_attr == "test_value"
+
+
+def test_chatgpt_config_streams_parallel_tool_calls_with_distinct_indices(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+):
+ monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
+ chunks: Final = (
+ _make_chunk(tool_calls=[_make_tc(index=0, id="call_aaa", name="get_weather")]),
+ _make_chunk(tool_calls=[_make_tc(index=0, id="call_bbb", name="get_time")]),
+ )
+
+ results: Final = list(ChatGPTConfig().post_stream_processing(iter(chunks)))
+
+ assert [result.choices[0].delta.tool_calls[0].index for result in results] == [0, 1]
From e276bf34f7035842d5365f3530bf22377848a41f Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 12:10:57 -0700
Subject: [PATCH 021/203] test(perf): set the small case's
postgres_max_background_statement_per_request ceiling to 0.9 (#47616)
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
tests/e2e/perf/cases/small_bodies_5rps.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/e2e/perf/cases/small_bodies_5rps.py b/tests/e2e/perf/cases/small_bodies_5rps.py
index 4bf58cde3a5..f4013b79eac 100644
--- a/tests/e2e/perf/cases/small_bodies_5rps.py
+++ b/tests/e2e/perf/cases/small_bodies_5rps.py
@@ -77,7 +77,7 @@ CASE: Final = PerformanceTestCase(
Limit("memory_growth_megabytes", max=100),
Limit("postgres_calls_per_request", max=1.31),
Limit("postgres_max_statement_per_request", max=0.27),
- Limit("postgres_max_background_statement_per_request", max=0.92),
+ Limit("postgres_max_background_statement_per_request", max=0.9),
Limit("redis_calls_per_request", max=13.8),
Limit("cpu_seconds_per_request", max=0.15),
),
From 423362bab26203fe613867497b01fc0579efdc0f Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 12:21:42 -0700
Subject: [PATCH 022/203] fix(proxy): flush the spend-log queue only for
Responses turns the bridge reads back (#47614)
Every /v1/responses spend log asked its worker to flush the queue right away so a
chained previous_response_id could find the row. Only the chat-completions bridge
reads that row. Native Responses providers hold the conversation upstream, so their
rows now wait for the batch write like chat rows do
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
litellm/proxy/db/db_spend_update_writer.py | 48 +++++++++++++++--
litellm/responses/main.py | 12 +++--
.../proxy/db/test_db_spend_update_writer.py | 51 +++++++++++++++----
3 files changed, 91 insertions(+), 20 deletions(-)
diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py
index 58a801eebe4..e7909cbe600 100644
--- a/litellm/proxy/db/db_spend_update_writer.py
+++ b/litellm/proxy/db/db_spend_update_writer.py
@@ -95,6 +95,7 @@ if TYPE_CHECKING:
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution
from litellm.proxy.utils import PrismaClient, ProxyLogging
+ from litellm.router import Router
else:
PrismaClient = Any
ProxyLogging = Any
@@ -122,6 +123,34 @@ def _parsed_request_tags(request_tags: object) -> Iterable[object] | TypeError |
return TypeError(f"'{type(parsed).__name__}' object is not iterable")
+def _turn_chains_through_the_db(payload: Mapping[str, object], llm_router: "Router | None") -> bool:
+ """A Responses turn the chat-completions bridge replays from its spend log on the next `previous_response_id`.
+
+ Native Responses providers hold the conversation themselves, so their rows can wait for the batch write.
+ A row with no router deployment behind it is treated as bridged, which keeps today's immediate flush.
+ """
+ if payload.get("call_type") not in RESPONSES_SESSION_CALL_TYPES:
+ return False
+ model_id: Final = payload.get("model_id")
+ deployment: Final = (
+ llm_router.get_deployment(model_id=model_id) if llm_router is not None and isinstance(model_id, str) else None
+ )
+ if deployment is None:
+ return True
+ from litellm.responses.main import provider_stores_the_conversation
+
+ litellm_params: Final = deployment.litellm_params
+ return not provider_stores_the_conversation(
+ litellm_params.model,
+ litellm_params.custom_llm_provider,
+ {
+ "use_chat_completions_api": litellm_params.use_chat_completions_api,
+ "model_info": deployment.model_info.model_dump(),
+ "api_base": litellm_params.api_base,
+ },
+ )
+
+
def _org_member_transaction_key(org_id: str, user_id: str) -> str:
return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}"
@@ -565,12 +594,13 @@ class DBSpendUpdateWriter:
get_request_model_access_groups,
)
+ llm_router: Final = get_llm_router()
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=completion_response,
start_time=start_time,
end_time=end_time,
- llm_router=get_llm_router(),
+ llm_router=llm_router,
)
payload["spend"] = response_cost or 0.0
if isinstance(payload["startTime"], datetime):
@@ -585,7 +615,10 @@ class DBSpendUpdateWriter:
payload["team_id"] = team_id
if not await self._record_spend_log(
- payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs
+ payload=payload,
+ prisma_client=prisma_client,
+ disable_spend_logs=disable_spend_logs,
+ llm_router=llm_router,
):
return False
@@ -649,14 +682,18 @@ class DBSpendUpdateWriter:
return True
async def _record_spend_log(
- self, payload: SpendLogsPayload, prisma_client: "PrismaClient | None", disable_spend_logs: bool
+ self,
+ payload: SpendLogsPayload,
+ prisma_client: "PrismaClient | None",
+ disable_spend_logs: bool,
+ llm_router: "Router | None" = None,
) -> bool:
if prisma_client is not None and _is_batch_cost_row(payload):
return await self._claim_batch_cost_spend_log(
payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs
)
if disable_spend_logs is False:
- await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client)
+ await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client, llm_router=llm_router)
return True
async def _claim_batch_cost_spend_log(
@@ -1534,6 +1571,7 @@ class DBSpendUpdateWriter:
payload: dict | SpendLogsPayload,
prisma_client: PrismaClient | None = None,
spend_logs_url: str | None = os.getenv("SPEND_LOGS_URL"),
+ llm_router: "Router | None" = None,
) -> PrismaClient | None:
verbose_proxy_logger.debug(
"Writing spend log to db - request_id: {}, spend: {}".format(
@@ -1544,7 +1582,7 @@ class DBSpendUpdateWriter:
from litellm.proxy.utils import enqueue_spend_logs, request_spend_log_flush
await enqueue_spend_logs(prisma_client, (payload,))
- if payload.get("call_type") in RESPONSES_SESSION_CALL_TYPES:
+ if _turn_chains_through_the_db(_SPEND_METADATA_ADAPTER.validate_python(payload), llm_router):
request_spend_log_flush(prisma_client)
else:
verbose_proxy_logger.debug("prisma_client is None. Skipping writing spend logs to db.")
diff --git a/litellm/responses/main.py b/litellm/responses/main.py
index 995921beed1..3bdb1c9c375 100644
--- a/litellm/responses/main.py
+++ b/litellm/responses/main.py
@@ -306,7 +306,7 @@ async def aresponses_api_with_mcp(
joins_request_trace
and not stream
and previous_response_id is None
- and not _provider_stores_the_first_round(model, custom_llm_provider, kwargs)
+ and not provider_stores_the_conversation(model, custom_llm_provider, kwargs)
)
listing_trace_id: Final = (
@@ -463,7 +463,7 @@ async def aresponses_api_with_mcp(
if tool_results:
chains_on_first_round: Final = not LiteLLM_Proxy_MCP_Handler.is_persistence_disabled(
call_params
- ) and _provider_stores_the_first_round(model, custom_llm_provider, kwargs)
+ ) and provider_stores_the_conversation(model, custom_llm_provider, kwargs)
follow_up_input: Final = LiteLLM_Proxy_MCP_Handler.create_follow_up_input(
response=response,
@@ -687,8 +687,12 @@ def _will_bridge_to_chat_completions(
)
-def _provider_stores_the_first_round(model: str, custom_llm_provider: str | None, kwargs: Mapping[str, object]) -> bool:
- """A bridged first MCP round runs as an internal call, so no spend log holds it for the follow-up to replay."""
+def provider_stores_the_conversation(model: str, custom_llm_provider: str | None, kwargs: Mapping[str, object]) -> bool:
+ """Whether the provider, not a LiteLLM spend log, holds this turn for a `previous_response_id` follow-up.
+
+ A bridged turn is replayed from its spend log, so the bridge needs that row in the DB, and a bridged
+ first MCP round runs as an internal call that writes no spend log at all.
+ """
api_base: Final = _api_base_kwarg(kwargs)
return not _will_bridge_to_chat_completions(
model,
diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py
index 1ec65a4fdcd..d440b99a632 100644
--- a/tests/unit/proxy/db/test_db_spend_update_writer.py
+++ b/tests/unit/proxy/db/test_db_spend_update_writer.py
@@ -4130,28 +4130,57 @@ async def test_commit_spend_updates_raises_after_exhausting_lock_timeout_retries
assert mock_prisma_client.db.tx.call_count == 3
+def _responses_router() -> litellm.Router:
+ return litellm.Router(
+ model_list=[
+ {
+ "model_name": "native",
+ "litellm_params": {"model": "openai/gpt-5.1", "api_key": "sk-test"},
+ "model_info": {"id": "dep-native"},
+ },
+ {
+ "model_name": "bridged",
+ "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"},
+ "model_info": {"id": "dep-bridged"},
+ },
+ {
+ "model_name": "forced-bridge",
+ "litellm_params": {"model": "openai/gpt-5.1", "api_key": "sk-test", "use_chat_completions_api": True},
+ "model_info": {"id": "dep-forced-bridge"},
+ },
+ ]
+ )
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
- "call_type, expects_flush",
- [("aresponses", True), ("responses", True), ("acompletion", False)],
+ "call_type, model_id, expects_flush",
+ [
+ ("aresponses", "dep-bridged", True),
+ ("responses", "dep-bridged", True),
+ ("aresponses", "dep-forced-bridge", True),
+ ("aresponses", None, True),
+ ("aresponses", "dep-native", False),
+ ("responses", "dep-native", False),
+ ("acompletion", "dep-bridged", False),
+ ],
)
-async def test_insert_spend_log_asks_for_an_immediate_flush_on_rows_other_workers_read_back(
- call_type: str, expects_flush: bool
+async def test_insert_spend_log_asks_for_an_immediate_flush_only_on_rows_the_bridge_reads_back(
+ call_type: str, model_id: str | None, expects_flush: bool
):
"""
- A `previous_response_id` chained straight off the previous turn reads the DB, so a
- Responses row cannot sit in this worker's queue until the monitor's next poll.
+ A `previous_response_id` chained straight off a bridged turn reads that turn's row from the
+ DB, so it cannot sit in this worker's queue until the monitor's next poll. A native Responses
+ provider holds the conversation itself, so its row waits for the batch write like a chat row.
"""
db_writer = DBSpendUpdateWriter()
prisma = _tool_usage_prisma()
prisma.spend_log_flush_requested = asyncio.Event()
+ payload = {"request_id": "req-1", "call_type": call_type, "model_id": model_id}
- await db_writer._insert_spend_log_to_db(
- payload={"request_id": "req-1", "call_type": call_type},
- prisma_client=prisma,
- )
+ await db_writer._insert_spend_log_to_db(payload=payload, prisma_client=prisma, llm_router=_responses_router())
- assert prisma.spend_log_transactions == [{"request_id": "req-1", "call_type": call_type}]
+ assert prisma.spend_log_transactions == [payload]
assert prisma.spend_log_flush_requested.is_set() is expects_flush
From d677f4a455e4476e10e29eadc23bc82c5e26d1b8 Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 12:24:19 -0700
Subject: [PATCH 023/203] fix(proxy): size pass rebuilds reset row statistics
with ANALYZE and keeps its reclaimable count (#47501)
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
.../db_transaction_queue/spend_log_cleanup.py | 93 ++++++++++++++++---
tests/unit/proxy/test_spend_log_cleanup.py | 44 +++++++--
2 files changed, 117 insertions(+), 20 deletions(-)
diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
index e1e1d3690ff..7f5efcdaec3 100644
--- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
+++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
@@ -121,10 +121,12 @@ def advance_size_state(state: SizeCleanupState | None, probe: StorageProbe) -> S
"""
Roll the reclaimable count forward to what the catalog shows now.
- Without a prior state, or after the cumulative counters went backwards (a statistics reset), the count
- starts from the dead rows Postgres still knows about, at the files' average bytes per row. A probe with no
- row counters at all (`stats_are_stale`) cannot seed anything: the prior count is carried forward, and with
- no prior count the files are taken as fully reclaimable so nothing is deleted on its say-so.
+ Without a prior state the count starts from the dead rows Postgres still knows about, at the files' average
+ bytes per row. After the cumulative counters went backwards (a statistics reset, e.g. a crash) the prior
+ count is carried forward less any shrink of the files, or the dead-row seed if that is larger, because the
+ reset changed the counters and not the files. A probe with no row counters at all (`stats_are_stale`)
+ cannot seed anything: the prior count is carried forward, and with no prior count the files are taken as
+ fully reclaimable so nothing is deleted on its say-so.
"""
if probe.stats_are_stale:
return SizeCleanupState(
@@ -138,10 +140,26 @@ def advance_size_state(state: SizeCleanupState | None, probe: StorageProbe) -> S
)
if state is None or probe.inserted_total < state.inserted_total or probe.deleted_total < state.deleted_total:
seed_bytes_per_row: Final = probe.allocated_bytes / max(probe.live_rows + probe.dead_rows, 1)
+ seeded: Final = min(probe.allocated_bytes, round(probe.dead_rows * seed_bytes_per_row))
+ if state is None:
+ return SizeCleanupState(
+ allocated_bytes=probe.allocated_bytes,
+ reclaimable_bytes=seeded,
+ bytes_per_row=max(seed_bytes_per_row, 1.0),
+ inserted_total=probe.inserted_total,
+ deleted_total=probe.deleted_total,
+ )
+ carried: Final = max(
+ seeded,
+ min(probe.allocated_bytes, state.reclaimable_bytes - max(0, state.allocated_bytes - probe.allocated_bytes)),
+ )
+ carried_live_bytes: Final = probe.allocated_bytes - carried
return SizeCleanupState(
allocated_bytes=probe.allocated_bytes,
- reclaimable_bytes=min(probe.allocated_bytes, round(probe.dead_rows * seed_bytes_per_row)),
- bytes_per_row=max(seed_bytes_per_row, 1.0),
+ reclaimable_bytes=carried,
+ bytes_per_row=max(carried_live_bytes / probe.live_rows, 1.0)
+ if probe.live_rows > 0
+ else state.bytes_per_row,
inserted_total=probe.inserted_total,
deleted_total=probe.deleted_total,
)
@@ -357,6 +375,11 @@ _STORAGE_SQL: Final = """
# The age job and the size job are scheduled on separate SpendLogCleanup instances, so the lock is per process.
_PASS_LOCK: Final = asyncio.Lock()
+_ANALYZE_SQL: Final = 'ANALYZE "LiteLLM_SpendLogs", "LiteLLM_SpendLogToolIndex"'
+# pg_stat_user_tables publishes what ANALYZE counted up to a second later (the stats collector on Postgres 14).
+_STATS_SETTLE_SECONDS: Final = 1.0
+_STATS_SETTLE_ATTEMPTS: Final = 5
+
SPEND_LOG_CLEANUP_BOUND_SETTINGS: Final = (
"maximum_spend_logs_cleanup_batch_size",
"maximum_spend_logs_cleanup_max_batches",
@@ -586,6 +609,48 @@ class SpendLogCleanup:
deleted_total=row.deleted_total,
)
+ async def _reprobe_after_analyze(
+ self, prisma_client: PrismaClient, deadline: float, stale: StorageProbe
+ ) -> StorageProbe | None:
+ """
+ Row counters that are all zero under a file with data were reset, e.g. by a crash. ANALYZE rebuilds
+ n_live_tup and n_dead_tup from a 30,000-row sample in seconds and blocks nothing, so the pass runs it and
+ reads the catalog again rather than trusting an estimate of zero or waiting a day for autoanalyze. The
+ new counts show up in pg_stat_user_tables a moment after ANALYZE returns, hence the settled re-reads.
+ """
+ verbose_proxy_logger.warning(
+ "Spend-log row statistics are stale (allocated=%s bytes, live_rows=0, dead_rows=0); running %s before "
+ "the size pass",
+ stale.allocated_bytes,
+ _ANALYZE_SQL,
+ )
+ if time.monotonic() >= deadline:
+ return None
+ timeout_ms: Final = self._timeout_ms(deadline)
+ try:
+ async with db_span("analyze", "LiteLLM_SpendLogs"), bounded_tx(prisma_client, timeout_ms) as tx:
+ await tx.execute_raw(trusted_sql(f"SET LOCAL statement_timeout = {timeout_ms}"))
+ await tx.execute_raw(trusted_sql(_ANALYZE_SQL))
+ except Exception as e: # noqa: BLE001 - a failed ANALYZE skips the size pass, it must not fail the run
+ verbose_proxy_logger.warning("Could not ANALYZE the spend-log tables: %s: %s", type(e).__name__, e)
+ return None
+ refreshed: Final = await self._probe_once_settled(prisma_client, deadline, _STATS_SETTLE_ATTEMPTS)
+ if refreshed is None or refreshed.stats_are_stale:
+ verbose_proxy_logger.warning(
+ "Spend-log row statistics are still stale after ANALYZE; skipping the size pass"
+ )
+ return None
+ return refreshed
+
+ async def _probe_once_settled(
+ self, prisma_client: PrismaClient, deadline: float, attempts_left: int
+ ) -> StorageProbe | None:
+ await asyncio.sleep(_STATS_SETTLE_SECONDS)
+ refreshed: Final = await self._probe_storage(prisma_client, deadline)
+ if refreshed is None or not refreshed.stats_are_stale or attempts_left <= 1:
+ return refreshed
+ return await self._probe_once_settled(prisma_client, deadline, attempts_left - 1)
+
async def _load_size_state(self, prisma_client: PrismaClient, deadline: float) -> SizeCleanupState | None:
"""The reclaimable count saved by the last check, or None on the first check or when it cannot be read."""
if time.monotonic() >= deadline:
@@ -1042,15 +1107,15 @@ class SpendLogCleanup:
second. The run's batch cap and budget apply to the whole pass, so a pass that stops short reports how
far above the target it still is and the next check continues from there.
"""
- probe: Final = await self._probe_storage(prisma_client, deadline)
- if probe is None:
+ measured: Final = await self._probe_storage(prisma_client, deadline)
+ if measured is None:
return None
- if probe.stats_are_stale:
- verbose_proxy_logger.warning(
- "Spend-log row statistics are stale (allocated=%s bytes, live_rows=0, dead_rows=0); skipping the "
- 'size pass until pg_stat_user_tables is repopulated, ANALYZE "LiteLLM_SpendLogs" refreshes it',
- probe.allocated_bytes,
- )
+ probe: Final = (
+ await self._reprobe_after_analyze(prisma_client, deadline, measured)
+ if measured.stats_are_stale
+ else measured
+ )
+ if probe is None:
return None
state: Final = advance_size_state(await self._load_size_state(prisma_client, deadline), probe)
before: Final = SpendLogStorage(
diff --git a/tests/unit/proxy/test_spend_log_cleanup.py b/tests/unit/proxy/test_spend_log_cleanup.py
index a0f222ed3e0..876ffe96584 100644
--- a/tests/unit/proxy/test_spend_log_cleanup.py
+++ b/tests/unit/proxy/test_spend_log_cleanup.py
@@ -1877,19 +1877,41 @@ async def test_size_pass_stops_at_the_run_batch_cap_and_reports_how_far_above_th
@pytest.mark.asyncio
-async def test_size_pass_skips_on_stale_row_statistics_instead_of_trusting_an_estimate_of_zero(caplog):
+async def test_size_pass_analyzes_and_measures_again_when_the_row_statistics_were_reset(caplog):
+ """A crash zeroes pg_stat_user_tables under a full file; the pass rebuilds the counters instead of skipping.
+ The first read after ANALYZE still shows zeros, as the stats collector publishes them a moment later"""
db = MagicMock()
_wire_tx(db)
- db.execute_raw = AsyncMock(return_value=1000)
- _wire_size_probes(db, [_storage(allocated=5 * 1024**3, live=0, dead=0)], cutoff=datetime.now(timezone.utc))
+ statements = AsyncMock(return_value=0)
+ db.execute_raw = statements
+ saves = _wire_size_probes(
+ db,
+ [
+ _storage(allocated=10 * 1024**3, live=0, dead=0, inserted=0, deleted=0),
+ _storage(allocated=10 * 1024**3, live=0, dead=0, inserted=0, deleted=0),
+ _storage(10 * 1024**3, 1000, 0, 0, 0),
+ ],
+ cutoff=datetime.now(timezone.utc),
+ saved_state=SizeCleanupState(
+ allocated_bytes=10 * 1024**3,
+ reclaimable_bytes=9 * 1024**3 + 512 * 1024**2,
+ bytes_per_row=1024.0,
+ inserted_total=50_000,
+ deleted_total=49_000,
+ ),
+ )
client = MagicMock()
client.db = db
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
await _size_cleaner({}).cleanup_spend_logs_by_size(client)
- assert db.deletes.await_count == 0
+ issued = [call.args[0] for call in statements.await_args_list]
+ assert issued == ['ANALYZE "LiteLLM_SpendLogs", "LiteLLM_SpendLogToolIndex"'], (
+ "the only statement is the ANALYZE: no DELETE, the carried count puts the table under 1GB"
+ )
assert any("row statistics are stale" in record.getMessage() for record in caplog.records)
+ assert saves[0].reclaimable_bytes == 9 * 1024**3 + 512 * 1024**2
@pytest.mark.asyncio
@@ -1957,9 +1979,19 @@ _RUN_STATE = SizeCleanupState(
),
("the retention job deleted 2000 more rows", _storage(32_030_720, 3000, 2000, 20_000, 17_000), 27_217_000),
(
- "statistics reset: counters restart from the dead rows",
+ "statistics reset (a crash, then ANALYZE): the count is carried forward, not restarted from the dead rows",
_storage(32_030_720, 5000, 1000, 5000, 1000),
- 5_338_453,
+ 24_015_000,
+ ),
+ (
+ "statistics reset on a file VACUUM FULL shrank meanwhile: the carried count loses the shrink",
+ _storage(8_015_720, 5000, 0, 5000, 0),
+ 0,
+ ),
+ (
+ "statistics reset with more dead rows than the carried count explains: the dead rows win",
+ _storage(32_030_720, 1000, 19_000, 1000, 19_000),
+ 30_429_184,
),
],
)
From f0ae445cc4567335d33a01030891fae94ef32467 Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 12:24:38 -0700
Subject: [PATCH 024/203] fix(proxy): bill partial Responses API streams when
the client disconnects (#47611)
* fix(proxy): bill partial Responses API streams when the client disconnects
Streamed /v1/responses requests and the default-route OpenAI and Azure /v1/messages requests behind them logged a 499 with zero spend when the client went away mid-stream, while /v1/chat/completions already billed the chunks it had sent. The Responses API iterator now keeps the response.created payload and can hand the disconnect cleanup a response.incomplete event with counted usage and the served service_tier, and the messages-route byte stream and router wrappers delegate to it.
* fix(proxy): satisfy lint gates for partial Responses API disconnect billing
Make the disconnect helper public so the proxy, router and messages adapter can import it without reportPrivateUsage, type the SSE byte stream as an AsyncGenerator so aclose is known, read litellm_call_id off the logging object, and keep the new tests on Final locals with the recorder injected through dynamic_async_success_callbacks instead of litellm.callbacks.
* fix(router): bill the primary stream's tokens after a mid-stream hop and chat-bridge Responses streams on disconnect
The router's fallback wrapper now folds the primary Responses stream's relayed tokens into the fallback's disconnect event the way _combine_responses_fallback_usage does for a natural end of stream, and LiteLLMCompletionStreamingIterator (chat-only providers served through /v1/responses) builds its disconnect event from the chat completion chunks it collected. stamp_responses_usage_cost is now public so the bridge module can price that event.
* Potential fix for pull request finding 'CodeQL / Log Injection'
Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
---------
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
---
.../responses_adapters/streaming_iterator.py | 42 ++++++-
litellm/proxy/common_request_processing.py | 12 ++
.../streaming_iterator.py | 39 +++++-
litellm/responses/streaming_iterator.py | 72 ++++++++++-
litellm/router.py | 26 ++++
.../proxy/test_common_request_processing.py | 112 ++++++++++++++++++
.../test_litellm_completion_responses.py | 37 ++++++
.../test_responses_streaming_iterator.py | 100 ++++++++++++++++
.../unit/responses/test_streaming_iterator.py | 20 ++--
tests/unit/test_router/test_router.py | 65 ++++++++++
10 files changed, 507 insertions(+), 18 deletions(-)
diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py
index 85287f5433a..e628a30fdd6 100644
--- a/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py
+++ b/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py
@@ -2,7 +2,7 @@
## Translates OpenAI call to Anthropic `/v1/messages` format
import asyncio
from collections import deque
-from collections.abc import AsyncIterator, Iterator, Mapping
+from collections.abc import AsyncGenerator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final
from pydantic import ConfigDict, field_validator
@@ -21,7 +21,10 @@ from litellm.llms.anthropic.pass_through.messages.utils import (
)
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
-from litellm.responses.streaming_iterator import stream_error_status_and_message
+from litellm.responses.streaming_iterator import (
+ partial_responses_incomplete_event_on_disconnect,
+ stream_error_status_and_message,
+)
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.base import LiteLLMBaseModel
@@ -33,6 +36,7 @@ from .transformation import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
+ from litellm.types.llms.openai import ResponseIncompleteEvent
class _UpstreamFailure(LiteLLMBaseModel):
@@ -488,8 +492,38 @@ class AnthropicResponsesStreamWrapper:
raise StopAsyncIteration
- async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]:
- """Yield SSE-encoded bytes for each Anthropic event chunk."""
+ def async_anthropic_sse_wrapper(self) -> "AnthropicResponsesSSEByteStream":
+ """SSE-encoded bytes for each Anthropic event chunk."""
+ return AnthropicResponsesSSEByteStream(self)
+
+ async def _sse_frames(self) -> AsyncGenerator[bytes, None]:
safeguard_results: Final = StreamedSafeguardResults(self.safeguards_evaluator)
async for chunk in self:
yield anthropic_sse_frame(await safeguard_results.observe(chunk))
+
+ def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
+ return partial_responses_incomplete_event_on_disconnect(self.responses_stream)
+
+
+class AnthropicResponsesSSEByteStream:
+ """
+ The byte stream the proxy relays for a default-route /v1/messages request.
+ Keeps the Responses API iterator reachable so a client disconnect can bill
+ the partial output the way native /v1/responses streams do.
+ """
+
+ def __init__(self, wrapper: AnthropicResponsesStreamWrapper) -> None:
+ self._wrapper: Final = wrapper
+ self._frames: Final = wrapper._sse_frames() # pyright: ignore[reportPrivateUsage] # the wrapper's own byte stream
+
+ def __aiter__(self) -> "AnthropicResponsesSSEByteStream":
+ return self
+
+ async def __anext__(self) -> bytes:
+ return await self._frames.__anext__()
+
+ async def aclose(self) -> None:
+ await self._frames.aclose()
+
+ def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
+ return self._wrapper.partial_response_on_disconnect()
diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index 24a7b46da4b..2ffdd02cdf5 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -128,6 +128,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
_check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
check_and_merge_model_level_guardrails,
)
+from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
from litellm.router import Router
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
@@ -455,6 +456,13 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
# A natural end-of-stream success event already fired and released the
# slot; do not bill again, and let the caller skip the slot release.
return True
+ partial_responses_event: Final = partial_responses_incomplete_event_on_disconnect(response)
+ if partial_responses_event is not None:
+ verbose_proxy_logger.debug(
+ "Billing partial streamed Responses API spend after client disconnect, litellm_call_id=%s",
+ sanitize_for_log(logging_obj.litellm_call_id),
+ )
+ return await _dispatch_disconnect_billing_event(logging_obj, partial_responses_event)
chunks: Final[object] = getattr(response, "chunks", None)
if not isinstance(chunks, list) or not chunks:
return False
@@ -484,6 +492,10 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
partial_usage: Final = getattr(partial_response, "usage", None)
if isinstance(partial_usage, Usage):
backfill_missing_cache_usage_fields(partial_usage)
+ return await _dispatch_disconnect_billing_event(logging_obj, partial_response)
+
+
+async def _dispatch_disconnect_billing_event(logging_obj: LiteLLMLoggingObj, partial_response: object) -> bool:
try:
await logging_obj.dispatch_success_handlers(
partial_response,
diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py
index 5c9e5cac5ab..84d4648f75f 100644
--- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py
+++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py
@@ -22,7 +22,7 @@ from litellm.responses.litellm_completion_transformation.tool_search import (
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
-from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
+from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import (
PART_UNION_TYPES,
@@ -1363,6 +1363,43 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
return False
return not any(getattr(part, "text", None) for part in getattr(item, "content", None) or ())
+ def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
+ """
+ The response.incomplete event to bill when the client went away mid-stream.
+ Built from the chat completion chunks relayed so far, so the usage is what
+ stream_chunk_builder counts for them.
+ """
+ if self.completed_response is not None or not self.collected_chat_completion_chunks:
+ return None
+ built: Final = stream_chunk_builder(
+ chunks=self.collected_chat_completion_chunks,
+ messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped
+ input=self.request_input,
+ responses_api_request=dict(self.responses_api_request),
+ ),
+ logging_obj=self.litellm_logging_obj,
+ )
+ if not isinstance(built, ModelResponse):
+ return None
+ responses_api_response: Final = (
+ LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped
+ request_input=self.request_input,
+ chat_completion_response=built,
+ responses_api_request=self.responses_api_request,
+ )
+ )
+ partial: Final = self._with_encoded_response_id(
+ responses_api_response.model_copy(
+ update={
+ "id": self._cached_response_id or responses_api_response.id,
+ "status": "incomplete",
+ "output": list(self._output_with_streamed_item_ids(responses_api_response)),
+ }
+ )
+ )
+ stamp_responses_usage_cost(partial, self.litellm_logging_obj)
+ return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial)
+
def _emit_terminal_response_event(
self, litellm_model_response: ModelResponse
) -> ResponseCompletedEvent | ResponseIncompleteEvent | None:
diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py
index bcd917c69d5..c5dc45566a9 100644
--- a/litellm/responses/streaming_iterator.py
+++ b/litellm/responses/streaming_iterator.py
@@ -58,10 +58,12 @@ from litellm.types.integrations.custom_logger import converted_stream_requested
from litellm.types.llms.openai import (
PART_UNION_TYPES,
ResponseAPIUsage,
+ ResponseIncompleteEvent,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
ResponsesAPIStreamingResponse,
)
+from litellm.types.responses.main import GenericResponseOutputItem, OutputText
from litellm.types.utils import CallTypes
from litellm.utils import async_post_call_success_deployment_hook
@@ -385,6 +387,7 @@ class BaseResponsesAPIStreamingIterator:
self._output_started = False
self._generated_content = ""
self._generated_tool_arguments = ""
+ self._created_response: ResponsesAPIResponse | None = None
self._completed_response_cached = False
self._completed_response_logged = False
self._completed_response_cache_hit: bool | None = None
@@ -544,6 +547,10 @@ class BaseResponsesAPIStreamingIterator:
# Store the completed response (also for incomplete/failed so logging still fires)
_chunk_type: Final = getattr(openai_responses_api_chunk, "type", None)
openai_types = get_openai_response_types()
+ if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED:
+ _created: Final[object] = getattr(openai_responses_api_chunk, "response", None)
+ if isinstance(_created, ResponsesAPIResponse):
+ self._created_response = _created
if openai_responses_api_chunk and _chunk_type in (
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
@@ -575,7 +582,7 @@ class BaseResponsesAPIStreamingIterator:
else openai_responses_api_chunk.model_copy(update={"response": _billed_response})
)
self.completed_response = _terminal_chunk
- _stamp_responses_usage_cost(_billed_response, self.logging_obj)
+ stamp_responses_usage_cost(_billed_response, self.logging_obj)
if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED:
self._handle_logging_failed_response()
@@ -1030,6 +1037,34 @@ class BaseResponsesAPIStreamingIterator:
if pending:
await asyncio.wait(pending)
+ def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
+ """
+ The response.incomplete event to bill when the client went away mid-stream,
+ the same shape the success handler assembles for a natural end of stream.
+ Its response is the response.created payload (it carries the served
+ service_tier) plus the text and tool arguments relayed so far, with counted
+ usage since the provider never sent any. None when nothing was relayed yet
+ or the stream already reached its terminal event and billed itself.
+ """
+ created: Final = self._created_response
+ generated: Final = self._generated_content + self._generated_tool_arguments
+ if self.completed_response is not None or created is None or not generated:
+ return None
+ usage: Final = _estimate_usage_safely(
+ self.model or "", self.request_data.get("input"), self.request_data, generated
+ )
+ if usage is None:
+ return None
+ partial: Final = created.model_copy(
+ update={
+ "status": "incomplete",
+ "usage": usage,
+ "output": _partial_output_items(self._generated_content),
+ }
+ )
+ stamp_responses_usage_cost(partial, self.logging_obj)
+ return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial)
+
def _note_yielded_event(self, event: ResponsesAPIStreamingResponse) -> None:
self._yielded_first_chunk = True
if event.type not in PRE_OUTPUT_LIFECYCLE_EVENT_TYPES:
@@ -1561,6 +1596,37 @@ def _add_text_like_part_events(
)
+@runtime_checkable
+class _BillsPartialStreamOnDisconnect(Protocol):
+ def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None: ...
+
+
+def partial_responses_incomplete_event_on_disconnect(stream: object) -> ResponseIncompleteEvent | None:
+ """Ask the stream behind a proxy or router wrapper for the partial response to bill, if it can build one."""
+ if not isinstance(stream, _BillsPartialStreamOnDisconnect):
+ return None
+ try:
+ partial: Final = stream.partial_response_on_disconnect()
+ except Exception as e: # noqa: BLE001 # disconnect billing is best-effort; never break stream teardown
+ verbose_logger.debug("Failed to build partial Responses API response for disconnect billing: %s", e)
+ return None
+ return partial if isinstance(partial, ResponseIncompleteEvent) else None
+
+
+def _partial_output_items(generated_text: str) -> Sequence[GenericResponseOutputItem]:
+ if not generated_text:
+ return []
+ return [
+ GenericResponseOutputItem(
+ type="message",
+ id=f"msg_{uuid.uuid4()}",
+ status="incomplete",
+ role="assistant",
+ content=[OutputText(type="output_text", text=generated_text, annotations=[])],
+ )
+ ]
+
+
def _billed_terminal_response(
response_obj: object, estimate: Callable[[], ResponseAPIUsage | None] | None
) -> ResponsesAPIResponse | None:
@@ -1667,7 +1733,7 @@ def _estimate_usage_safely(
return None
-def _stamp_responses_usage_cost(
+def stamp_responses_usage_cost(
response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None
) -> None:
if response_obj is None or logging_obj is None:
@@ -1717,7 +1783,7 @@ def build_synthetic_response_events(
chunk_size: int,
) -> list[ResponsesAPIStreamingResponse]:
openai_types: Final = get_openai_response_types()
- _stamp_responses_usage_cost(transformed, logging_obj)
+ stamp_responses_usage_cost(transformed, logging_obj)
events: Final[list[ResponsesAPIStreamingResponse]] = [
_build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed),
diff --git a/litellm/router.py b/litellm/router.py
index ac0933e158b..e04d7da8806 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -478,6 +478,7 @@ if TYPE_CHECKING:
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
from litellm.types.llms.openai import (
ResponseAPIUsage,
+ ResponseIncompleteEvent,
ResponseInputParam,
ResponsesAPIResponse,
)
@@ -983,6 +984,11 @@ class FallbackAwareAnthropicMessagesStream:
self._source_iterator = fallback_response
self.fallback_headers_adopted = True
+ def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
+ from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
+
+ return partial_responses_incomplete_event_on_disconnect(self._source_iterator)
+
def follow_source_attribution(self) -> None:
"""
A retry's or a fallback's stream carries a wrapper of its own, so a hop it makes before its
@@ -3911,6 +3917,7 @@ class Router:
from datetime import datetime
self._async_generator = async_generator
+ self._billing_source: object = source_iterator
# Mirror every attribute BaseResponsesAPIStreamingIterator.__init__
# would have set. The wrapper bypasses super().__init__ (it has no
# httpx.Response of its own and no provider config to drive), so
@@ -3959,10 +3966,29 @@ class Router:
def adopt_fallback_headers(self, fallback_response: object) -> tuple[dict[str, object], dict[str, object]]:
prepared: Final = Router._prepare_fallback_hidden_params(fallback_response)
+ self._billing_source = fallback_response
self._hidden_params = {**prepared[0], "additional_headers": prepared[1]}
self.fallback_headers_adopted = True
return prepared
+ def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
+ """
+ The event to bill on a client disconnect. After a mid-stream hop the primary
+ stream's relayed tokens are folded into the fallback's event, the way
+ _combine_responses_fallback_usage does for a natural end of stream.
+ """
+ from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
+
+ current: Final = partial_responses_incomplete_event_on_disconnect(self._billing_source)
+ if self._billing_source is source_iterator:
+ return current
+ primary: Final = partial_responses_incomplete_event_on_disconnect(source_iterator)
+ if current is None or primary is None:
+ return current or primary
+ if primary.response.usage is not None:
+ Router._combine_responses_fallback_usage(current, primary.response.usage)
+ return current
+
def __aiter__(self):
return self
diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py
index f837dc3fb5f..39423939b68 100644
--- a/tests/unit/proxy/test_common_request_processing.py
+++ b/tests/unit/proxy/test_common_request_processing.py
@@ -9011,6 +9011,118 @@ def _apply_stream_usage_tracking(
)
+ async def _start_partial_responses_stream(self, recorder: _RecordingSuccessLogger, *, relay_text: bool = True):
+ from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+ from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
+
+ mocked: Final = await litellm.aresponses(
+ model="gpt-4o-mini",
+ input="tell me a story",
+ mock_response="The codename is AZURE-FALCON-42 and the story is long.",
+ stream=True,
+ api_key="test-key",
+ )
+ logging_obj: Final = mocked.logging_obj
+ logging_obj.dynamic_async_success_callbacks = [recorder]
+ stream: Final = ResponsesAPIStreamingIterator(
+ response=httpx.Response(200),
+ model="gpt-4o-mini",
+ responses_api_provider_config=OpenAIResponsesAPIConfig(),
+ logging_obj=logging_obj,
+ request_data={"model": "gpt-4o-mini", "input": "tell me a story"},
+ custom_llm_provider="openai",
+ )
+ created: Final = {
+ "type": "response.created",
+ "sequence_number": 0,
+ "response": {
+ "id": "resp_partial",
+ "object": "response",
+ "created_at": 1700000000.0,
+ "model": "gpt-4o-mini",
+ "status": "in_progress",
+ "output": [],
+ "parallel_tool_calls": True,
+ "tool_choice": "auto",
+ "tools": [],
+ "service_tier": "priority",
+ },
+ }
+ delta: Final = {
+ "type": "response.output_text.delta",
+ "sequence_number": 1,
+ "item_id": "msg_partial",
+ "output_index": 0,
+ "content_index": 0,
+ "delta": "The codename is AZURE-FALCON-42",
+ }
+ for event in (created, delta) if relay_text else (created,):
+ stream._process_chunk(json.dumps(event))
+ return stream
+
+ async def _bill_and_wait(self, recorder: _RecordingSuccessLogger, streamed: object, logging_obj: object) -> bool:
+ billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, streamed)
+ for _ in range(50):
+ if recorder.success_events:
+ break
+ await asyncio.sleep(0.1)
+ await asyncio.sleep(0.5)
+ return billed
+
+ @pytest.mark.asyncio
+ async def test_disconnect_bills_partial_native_responses_stream_with_counted_usage(self):
+ from litellm.types.llms.openai import ResponsesAPIResponse
+
+ recorder: Final = _RecordingSuccessLogger()
+ stream: Final = await self._start_partial_responses_stream(recorder)
+ billed: Final = await self._bill_and_wait(recorder, stream, stream.logging_obj)
+
+ assert billed is True
+ assert len(recorder.success_events) == 1
+ partial_response: Final = recorder.success_events[0]["response_obj"]
+ assert isinstance(partial_response, ResponsesAPIResponse)
+ assert partial_response.status == "incomplete"
+ assert getattr(partial_response, "service_tier") == "priority"
+ assert partial_response.usage.output_tokens == litellm.token_counter(
+ model="gpt-4o-mini", text="The codename is AZURE-FALCON-42"
+ )
+ assert partial_response.usage.input_tokens > 0
+
+ @pytest.mark.asyncio
+ async def test_disconnect_bills_partial_default_route_messages_stream_through_the_responses_adapter(self):
+ from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import (
+ AnthropicResponsesStreamWrapper,
+ )
+ from litellm.router import FallbackAwareAnthropicMessagesStream
+ from litellm.types.llms.openai import ResponsesAPIResponse
+
+ async def _sse_frames() -> AsyncGenerator[bytes, None]:
+ yield b"event: message_start\n\n"
+
+ recorder: Final = _RecordingSuccessLogger()
+ stream: Final = await self._start_partial_responses_stream(recorder)
+ wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=stream, model="gpt-4o-mini")
+ streamed: Final = FallbackAwareAnthropicMessagesStream(_sse_frames(), wrapper.async_anthropic_sse_wrapper())
+ billed: Final = await self._bill_and_wait(recorder, streamed, stream.logging_obj)
+
+ assert billed is True
+ assert len(recorder.success_events) == 1
+ partial_response: Final = recorder.success_events[0]["response_obj"]
+ assert isinstance(partial_response, ResponsesAPIResponse)
+ assert partial_response.usage.output_tokens == litellm.token_counter(
+ model="gpt-4o-mini", text="The codename is AZURE-FALCON-42"
+ )
+
+ @pytest.mark.asyncio
+ async def test_disconnect_before_any_responses_output_bills_nothing(self):
+ recorder: Final = _RecordingSuccessLogger()
+ stream: Final = await self._start_partial_responses_stream(recorder, relay_text=False)
+ billed: Final = await self._bill_and_wait(recorder, stream, stream.logging_obj)
+
+ assert billed is False
+ assert recorder.success_events == []
+
+
class TestApplyStreamUsageTracking:
@pytest.mark.parametrize(
("general_settings", "requested_stream_options", "expected_stream_options"),
diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
index cc5d5c62739..f34ca7d0ff8 100644
--- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
+++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
@@ -6566,3 +6566,40 @@ def test_a_continued_chain_leaves_the_turns_metadata_alone_without_request_corre
assert request["litellm_trace_id"] == "chain-session"
assert "litellm_metadata" not in request
assert request["metadata"] == {CALLER_SESSION_ID_METADATA_KEY: "turn-session", "user_api_key": "hashed"}
+
+
+@pytest.mark.asyncio
+async def test_bridge_partial_response_on_disconnect_bills_the_relayed_chat_chunks():
+ from litellm.responses.litellm_completion_transformation.streaming_iterator import (
+ LiteLLMCompletionStreamingIterator,
+ )
+
+ chat_stream: Final = await litellm.acompletion(
+ model="gpt-4o-mini",
+ messages=[{"role": "user", "content": "tell me a story"}],
+ mock_response="The codename is AZURE-FALCON-42 and the story is long.",
+ stream=True,
+ api_key="test-key",
+ )
+ iterator: Final = LiteLLMCompletionStreamingIterator(
+ model="gpt-4o-mini",
+ litellm_custom_stream_wrapper=chat_stream,
+ request_input="tell me a story",
+ responses_api_request={},
+ custom_llm_provider="openai",
+ )
+ assert iterator.partial_response_on_disconnect() is None, "nothing relayed yet, nothing to bill"
+ events: Final = [await iterator.__anext__() for _ in range(6)]
+ relayed_text: Final = "".join(
+ getattr(event, "delta", "") for event in events if event.type == "response.output_text.delta"
+ )
+ assert relayed_text, "the bridge should have relayed some text by now"
+
+ incomplete_event: Final = iterator.partial_response_on_disconnect()
+
+ assert incomplete_event is not None
+ assert incomplete_event.type == "response.incomplete"
+ assert incomplete_event.response.status == "incomplete"
+ assert incomplete_event.response.usage is not None
+ assert incomplete_event.response.usage.output_tokens == litellm.token_counter(model="gpt-4o-mini", text=relayed_text)
+ assert incomplete_event.response.usage.input_tokens > 0
diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py
index 0419fce8ec2..37a58c32f38 100644
--- a/tests/unit/responses/test_responses_streaming_iterator.py
+++ b/tests/unit/responses/test_responses_streaming_iterator.py
@@ -7,7 +7,10 @@ crashed with exit 139 whenever any CustomLogger was registered).
"""
import asyncio
+import json
import time
+from collections.abc import Mapping
+from typing import Final
import httpx
import pytest
@@ -17,6 +20,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module
from litellm.responses import streaming_iterator as responses_streaming_iterator_module
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
from litellm.types.llms.openai import ResponsesAPIResponse
@@ -163,3 +167,99 @@ async def test_sync_callbacks_run_only_after_async_handler_completes(recording_e
submit_times = recording_executor.submit_times_for(logging_obj)
assert len(submit_times) == 1
assert submit_times[0] >= recorder.async_hook_finished
+
+
+_CREATED_RESPONSE_WITH_PRIORITY_TIER: Final[Mapping[str, object]] = {
+ "id": "resp_partial",
+ "object": "response",
+ "created_at": 1700000000.0,
+ "model": "gpt-5.4-nano",
+ "status": "in_progress",
+ "output": [],
+ "parallel_tool_calls": True,
+ "tool_choice": "auto",
+ "tools": [],
+ "service_tier": "priority",
+}
+_CREATED_WITH_PRIORITY_TIER: Final[Mapping[str, object]] = {
+ "type": "response.created",
+ "sequence_number": 0,
+ "response": _CREATED_RESPONSE_WITH_PRIORITY_TIER,
+}
+
+
+def _text_delta(text: str, sequence_number: int) -> Mapping[str, object]:
+ return {
+ "type": "response.output_text.delta",
+ "sequence_number": sequence_number,
+ "item_id": "msg_partial",
+ "output_index": 0,
+ "content_index": 0,
+ "delta": text,
+ }
+
+
+def _completed_after(text: str) -> Mapping[str, object]:
+ return {
+ "type": "response.completed",
+ "sequence_number": 9,
+ "response": {
+ **_CREATED_RESPONSE_WITH_PRIORITY_TIER,
+ "status": "completed",
+ "usage": {"input_tokens": 5, "output_tokens": 7, "total_tokens": 12},
+ },
+ }
+
+
+def _live_iterator(logging_obj: LitellmLogging) -> ResponsesAPIStreamingIterator:
+ return ResponsesAPIStreamingIterator(
+ response=httpx.Response(200),
+ model="gpt-5.4-nano",
+ responses_api_provider_config=OpenAIResponsesAPIConfig(),
+ logging_obj=logging_obj,
+ request_data={"model": "gpt-5.4-nano", "input": "tell me a story"},
+ custom_llm_provider="openai",
+ )
+
+
+def test_partial_response_on_disconnect_bills_the_relayed_text_at_the_served_service_tier():
+ iterator: Final = _live_iterator(_make_logging_obj())
+ for event in (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time ", 1), _text_delta("a fox slept.", 2)):
+ iterator._process_chunk(json.dumps(event))
+
+ incomplete_event: Final = iterator.partial_response_on_disconnect()
+
+ assert incomplete_event is not None
+ assert incomplete_event.type == "response.incomplete"
+ partial: Final = incomplete_event.response
+ assert partial.status == "incomplete"
+ assert getattr(partial, "service_tier") == "priority"
+ assert partial.output[0].content[0].text == "Once upon a time a fox slept."
+ assert partial.usage.output_tokens == litellm.token_counter(
+ model="gpt-5.4-nano", text="Once upon a time a fox slept."
+ )
+ assert partial.usage.input_tokens == litellm.token_counter(
+ model="gpt-5.4-nano", messages=[{"role": "user", "content": "tell me a story"}]
+ )
+ assert partial.usage.total_tokens == partial.usage.input_tokens + partial.usage.output_tokens
+
+
+@pytest.mark.parametrize(
+ "events",
+ [
+ (),
+ (_CREATED_WITH_PRIORITY_TIER,),
+ (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time", 1), _completed_after("Once upon a time")),
+ ],
+ ids=["nothing_relayed", "created_only", "already_completed"],
+)
+@pytest.mark.asyncio
+async def test_partial_response_on_disconnect_is_none_when_there_is_nothing_to_bill_or_the_stream_billed_itself(
+ events: tuple[Mapping[str, object], ...],
+):
+ iterator: Final = _live_iterator(_make_logging_obj())
+ for event in events:
+ iterator._process_chunk(json.dumps(event))
+
+ assert iterator.partial_response_on_disconnect() is None
+ await iterator._await_pending_logging()
diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py
index d0e07e1cc23..62d89a55e85 100644
--- a/tests/unit/responses/test_streaming_iterator.py
+++ b/tests/unit/responses/test_streaming_iterator.py
@@ -839,26 +839,26 @@ def _responses_api_response_with_usage() -> ResponsesAPIResponse:
def test_stamp_responses_usage_cost_stamps_computed_cost():
- from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
+ from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.response_cost_calculator.return_value = 0.000704
- _stamp_responses_usage_cost(response, logging_obj)
+ stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.000704)
logging_obj.response_cost_calculator.assert_called_once_with(result=response)
def test_stamp_responses_usage_cost_keeps_provider_reported_cost():
- from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
+ from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
setattr(response.usage, "cost", 0.5)
logging_obj = Mock(spec=LiteLLMLoggingObj)
- _stamp_responses_usage_cost(response, logging_obj)
+ stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.5)
logging_obj.response_cost_calculator.assert_not_called()
@@ -878,7 +878,7 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse:
def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
- from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
+ from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _unvalidated_response_with_dict_usage(
{
"input_tokens": 29,
@@ -890,7 +890,7 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
)
logging_obj = Mock(spec=LiteLLMLoggingObj)
- _stamp_responses_usage_cost(response, logging_obj)
+ stamp_responses_usage_cost(response, logging_obj)
assert isinstance(response.usage, ResponseAPIUsage)
assert response.usage.cost == pytest.approx(3e-05)
@@ -899,12 +899,12 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost():
- from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
+ from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149})
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.response_cost_calculator.return_value = 0.000704
- _stamp_responses_usage_cost(response, logging_obj)
+ stamp_responses_usage_cost(response, logging_obj)
assert isinstance(response.usage, ResponseAPIUsage)
assert response.usage.cost == pytest.approx(0.000704)
@@ -912,13 +912,13 @@ def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost():
def test_stamp_responses_usage_cost_survives_calculator_failure():
- from litellm.responses.streaming_iterator import _stamp_responses_usage_cost
+ from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.response_cost_calculator.side_effect = RuntimeError("cost map unavailable")
- _stamp_responses_usage_cost(response, logging_obj)
+ stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) is None
diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py
index 4a9dd199c08..d645ab4566d 100644
--- a/tests/unit/test_router/test_router.py
+++ b/tests/unit/test_router/test_router.py
@@ -29668,3 +29668,68 @@ def test_pre_call_checks_ignores_an_image_response_format_string(monkeypatch: py
)
assert [d["model_info"]["id"] for d in filtered] == ["only"]
+
+
+@pytest.mark.asyncio
+async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_streams_tokens():
+ """After a mid-stream hop, a disconnect bills the primary's relayed text plus the fallback's, not the fallback alone."""
+ import json
+
+ from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+ from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
+
+ def _relayed_iterator(text: str) -> ResponsesAPIStreamingIterator:
+ logging_obj: Final = MagicMock(spec=LiteLLMLogging)
+ logging_obj.model_call_details = {"litellm_params": {}}
+ logging_obj.completion_start_time = None
+ iterator: Final = ResponsesAPIStreamingIterator(
+ response=httpx.Response(200),
+ model="gpt-5.4",
+ responses_api_provider_config=OpenAIResponsesAPIConfig(),
+ logging_obj=logging_obj,
+ request_data={"model": "gpt-5.4", "input": "Hi"},
+ custom_llm_provider="openai",
+ )
+ created: Final = {
+ "id": "resp_1",
+ "object": "response",
+ "created_at": 1700000000.0,
+ "model": "gpt-5.4",
+ "status": "in_progress",
+ "output": [],
+ "parallel_tool_calls": True,
+ "tool_choice": "auto",
+ "tools": [],
+ }
+ iterator._process_chunk(json.dumps({"type": "response.created", "sequence_number": 0, "response": created}))
+ iterator._process_chunk(
+ json.dumps(
+ {
+ "type": "response.output_text.delta",
+ "sequence_number": 1,
+ "item_id": "msg_1",
+ "output_index": 0,
+ "content_index": 0,
+ "delta": text,
+ }
+ )
+ )
+ return iterator
+
+ router: Final = Router(
+ model_list=[{"model_name": "primary", "litellm_params": {"model": "openai/gpt-5.4", "api_key": "k"}}]
+ )
+ primary: Final = _relayed_iterator("Once upon a time there was a lighthouse keeper")
+ fallback: Final = _relayed_iterator("who lit the lamp")
+ wrapped: Final = await router._aresponses_streaming_iterator(
+ response=primary,
+ initial_kwargs={"model": "primary", "stream": True, "input": "Hi", "original_generic_function": AsyncMock()},
+ )
+ primary_tokens: Final = primary.partial_response_on_disconnect().response.usage.output_tokens
+ fallback_tokens: Final = fallback.partial_response_on_disconnect().response.usage.output_tokens
+
+ assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens
+
+ wrapped.adopt_fallback_headers(fallback)
+
+ assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens
From c6fc5b01bb0033cf189734b415e1f4dc85c79513 Mon Sep 17 00:00:00 2001
From: Priyansh Nandwana <136072927+msdianprince-7@users.noreply.github.com>
Date: Wed, 21 Oct 2026 01:00:16 +0530
Subject: [PATCH 025/203] fix(bedrock): send batch jobs to a configured
control-plane endpoint (#47260)
* fix(bedrock): send batch jobs to a configured control-plane endpoint
Bedrock batch create, retrieve and cancel always built the regional
bedrock.{region} host, so a VPC endpoint or compatible gateway could not
be used. Add aws_bedrock_control_plane_endpoint as a deployment setting
read by the signed HTTP path and the boto3 path, carry it through the
proxy credential filter, and refuse it in request bodies like the other
endpoint overrides. api_base stays out of it because on a Bedrock
deployment it points at the runtime host.
Fixes #45056
* fix(bedrock): read the control-plane endpoint from typed kwargs in job status
The named parameter added one unknown-argument error in litellm/batches/main.py,
which unpacks untyped kwargs into this function
* docs(bedrock): explain batch control-plane endpoint configuration
* fix(bedrock): keep the batch control-plane endpoint out of chat request bodies
A deployment that sets aws_bedrock_control_plane_endpoint also serves chat,
and the unregistered key was swept into the Bedrock request body, so every
chat call on it failed with "Extra inputs are not permitted". Register it
with the other Bedrock batch connection fields so it stays a LiteLLM-owned
kwarg on every route while batches still read it from litellm_params.
* test(bedrock): verify endpoint signatures without internal mocks
---------
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
---
.../batch_api/bedrock/README.md | 41 +++++++
.../litellm_core_utils/get_litellm_params.py | 1 +
litellm/llms/bedrock/batches/handler.py | 26 +++-
.../llms/bedrock/batches/transformation.py | 22 ++--
litellm/llms/bedrock/common_utils.py | 19 +++
litellm/proxy/auth/auth_utils.py | 1 +
litellm/types/litellm_params.py | 1 +
litellm/types/router.py | 1 +
.../unit/llms/bedrock/batches/test_handler.py | 42 +++++++
.../bedrock/batches/test_transformation.py | 115 +++++++++++++++++-
.../chat/test_converse_transformation.py | 50 ++++++++
tests/unit/proxy/auth/test_auth_utils.py | 1 +
tests/unit/types/test_litellm_params.py | 3 +
ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +
14 files changed, 310 insertions(+), 17 deletions(-)
create mode 100644 cookbook/litellm_proxy_server/batch_api/bedrock/README.md
diff --git a/cookbook/litellm_proxy_server/batch_api/bedrock/README.md b/cookbook/litellm_proxy_server/batch_api/bedrock/README.md
new file mode 100644
index 00000000000..1410d76658e
--- /dev/null
+++ b/cookbook/litellm_proxy_server/batch_api/bedrock/README.md
@@ -0,0 +1,41 @@
+# Bedrock batch endpoints
+
+Set `aws_bedrock_control_plane_endpoint` on a deployment when batch create, retrieve, and cancel requests must use a custom Bedrock control-plane endpoint. The proxy signs these requests for the configured URL. Without this setting, batch requests use the standard regional Bedrock endpoint
+
+```yaml
+model_list:
+ - model_name: bedrock-batch
+ litellm_params:
+ model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
+ aws_region_name: us-east-1
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN
+ s3_output_bucket_name: os.environ/AWS_S3_OUTPUT_BUCKET_NAME
+ aws_bedrock_control_plane_endpoint: os.environ/BEDROCK_CONTROL_PLANE_ENDPOINT
+```
+
+Set `BEDROCK_CONTROL_PLANE_ENDPOINT` to your Bedrock control-plane VPC endpoint or compatible gateway base URL, without `/model-invocation-job`. The proxy adds the operation path. The endpoint must be reachable from the proxy, and the configured AWS region must match the endpoint's signing region
+
+`api_base` and `aws_bedrock_runtime_endpoint` configure runtime inference separately. They do not override the batch control-plane endpoint. `s3_endpoint_url` configures S3 file access separately
+
+The control-plane setting belongs in deployment configuration. Sending `aws_bedrock_control_plane_endpoint` in an API request body is rejected. There is no automatic `AWS_BEDROCK_CONTROL_PLANE_ENDPOINT` environment fallback, so reference the desired environment variable explicitly as shown above
+
+Start the proxy with this configuration and set `LITELLM_API_BASE` to its URL. Set `BEDROCK_BATCH_INPUT_URI` to a valid input JSONL file in S3 that the batch service role can read
+
+```bash
+curl -sS "$LITELLM_API_BASE/v1/batches" \
+ -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+ -H 'Content-Type: application/json' \
+ -d "{\"model\":\"bedrock-batch\",\"input_file_id\":\"$BEDROCK_BATCH_INPUT_URI\",\"endpoint\":\"/v1/chat/completions\",\"completion_window\":\"24h\"}"
+```
+
+Use the returned `id` as `BATCH_ID` to retrieve or cancel the same job through the deployment's configured endpoint
+
+```bash
+curl -sS "$LITELLM_API_BASE/v1/batches/$BATCH_ID?model=bedrock-batch" \
+ -H "Authorization: Bearer $LITELLM_MASTER_KEY"
+
+curl -sS -X POST "$LITELLM_API_BASE/v1/batches/$BATCH_ID/cancel?model=bedrock-batch" \
+ -H "Authorization: Bearer $LITELLM_MASTER_KEY"
+```
diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py
index 6ca02cbbe64..657fd3abfbc 100644
--- a/litellm/litellm_core_utils/get_litellm_params.py
+++ b/litellm/litellm_core_utils/get_litellm_params.py
@@ -27,6 +27,7 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
"aws_external_id",
"aws_session_tags",
"aws_bedrock_runtime_endpoint",
+ "aws_bedrock_control_plane_endpoint",
"aws_bedrock_project_id",
}
)
diff --git a/litellm/llms/bedrock/batches/handler.py b/litellm/llms/bedrock/batches/handler.py
index 81057a1ee23..48e79d39b8e 100644
--- a/litellm/llms/bedrock/batches/handler.py
+++ b/litellm/llms/bedrock/batches/handler.py
@@ -5,7 +5,10 @@ from typing import TYPE_CHECKING, Final, Literal
from openai.types.batch import BatchRequestCounts
from openai.types.batch import Metadata as OpenAIBatchMetadata
-from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
+from litellm.llms.bedrock.common_utils import (
+ bedrock_control_plane_base_url,
+ configured_bedrock_control_plane_endpoint,
+)
from litellm.types.llms.bedrock import AwsAuthParams, AwsSessionTag
from litellm.types.utils import LiteLLMBatch
@@ -156,10 +159,14 @@ class BedrockBatchesHandler:
aws_session_tags=aws_session_tags,
)
creds: Final = BedrockBatchesConfig().resolve_credentials(auth_params, region)
+ control_plane_endpoint: Final = configured_bedrock_control_plane_endpoint(
+ kwargs.get("aws_bedrock_control_plane_endpoint")
+ )
client: Final = boto3.client(
"bedrock",
region_name=region,
+ endpoint_url=control_plane_endpoint,
aws_access_key_id=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_session_token=creds.token,
@@ -171,6 +178,7 @@ class BedrockBatchesHandler:
batch_id=batch_id,
aws_region_name=region,
logging_obj=logging_obj,
+ aws_bedrock_control_plane_endpoint=control_plane_endpoint,
**auth_params.model_dump(),
)
@@ -270,7 +278,7 @@ class BedrockBatchesHandler:
batch_id: str,
aws_region_name: str | None = None,
logging_obj=None,
- **kwargs,
+ **kwargs: object,
) -> "LiteLLMBatch":
"""
Handle ``GetModelInvocationJob`` status check for AWS Bedrock bulk batch
@@ -294,7 +302,10 @@ class BedrockBatchesHandler:
``aws_session_token``, ``aws_profile_name``,
``aws_role_name``, ``aws_session_name``,
``aws_web_identity_token``, ``aws_sts_endpoint``,
- ``aws_external_id``, ``aws_session_tags``). Unknown keys are ignored.
+ ``aws_external_id``, ``aws_session_tags``), plus
+ ``aws_bedrock_control_plane_endpoint`` to send the call to a VPC
+ endpoint or gateway instead of the regional ``bedrock`` host.
+ Unknown keys are ignored.
Returns:
``LiteLLMBatch`` shaped like an OpenAI Batch resource.
@@ -318,10 +329,14 @@ class BedrockBatchesHandler:
from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig
creds: Final = BedrockBatchesConfig().resolve_credentials(AwsAuthParams.model_validate(kwargs), region)
+ control_plane_endpoint: Final = configured_bedrock_control_plane_endpoint(
+ kwargs.get("aws_bedrock_control_plane_endpoint")
+ )
client: Final = boto3.client(
"bedrock",
region_name=region,
+ endpoint_url=control_plane_endpoint,
aws_access_key_id=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_session_token=creds.token,
@@ -334,14 +349,13 @@ class BedrockBatchesHandler:
# `GetModelInvocationJob` accepts either form, but only the bare id
# produces a sensible-looking URL in logs.
url_path_id: Final = _extract_job_id_from_arn(batch_id) or batch_id
+ control_plane_base: Final = bedrock_control_plane_base_url(control_plane_endpoint, region)
logging_obj.pre_call(
input=batch_id,
api_key="",
additional_args={
"complete_input_dict": {"jobIdentifier": batch_id},
- "api_base": (
- f"https://bedrock.{region}.{get_aws_dns_suffix(region)}/model-invocation-job/{url_path_id}"
- ),
+ "api_base": f"{control_plane_base}/model-invocation-job/{url_path_id}",
},
)
diff --git a/litellm/llms/bedrock/batches/transformation.py b/litellm/llms/bedrock/batches/transformation.py
index 50fb911709a..84f40bec9ca 100644
--- a/litellm/llms/bedrock/batches/transformation.py
+++ b/litellm/llms/bedrock/batches/transformation.py
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Final, Literal, cast
from httpx import Headers, Response
from pydantic import TypeAdapter, ValidationError
-from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix, is_bedrock_arn
+from litellm.litellm_core_utils.aws_partition import is_bedrock_arn
from litellm.litellm_core_utils.cloud_storage_security import (
BEDROCK_MANAGED_S3_BATCH_PREFIX,
)
@@ -32,6 +32,7 @@ from litellm.types.utils import LiteLLMBatch, LlmProviders, Usage
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import (
CommonBatchFilesUtils,
+ bedrock_control_plane_base_url,
merge_bedrock_aws_request_params,
resolve_s3_bucket_owner,
resolve_s3_encryption_key_id,
@@ -177,11 +178,10 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
aws_region_name: Final = self._get_aws_region_name(request_params, model)
- # Bedrock model invocation job endpoint
- # Format: https://bedrock.{region}.{partition dns suffix}/model-invocation-job
- bedrock_endpoint: Final = (
- f"https://bedrock.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/model-invocation-job"
+ control_plane_base: Final = bedrock_control_plane_base_url(
+ request_params.get("aws_bedrock_control_plane_endpoint"), aws_region_name
)
+ bedrock_endpoint: Final = f"{control_plane_base}/model-invocation-job"
return bedrock_endpoint
@@ -279,9 +279,10 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
# Use common utility for AWS signing
request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
aws_region_name: Final = self._get_aws_region_name(request_params, model)
- endpoint_url: Final = (
- f"https://bedrock.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/model-invocation-job"
+ control_plane_base: Final = bedrock_control_plane_base_url(
+ request_params.get("aws_bedrock_control_plane_endpoint"), aws_region_name
)
+ endpoint_url: Final = f"{control_plane_base}/model-invocation-job"
signed_headers, signed_data = self.common_utils.sign_aws_request(
service_name="bedrock",
data=bedrock_request,
@@ -431,12 +432,13 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
import urllib.parse as _ul
encoded_arn: Final = _ul.quote(batch_id, safe="")
- endpoint_url: Final = (
- f"https://bedrock.{region}.{get_aws_dns_suffix(region)}/model-invocation-job/{encoded_arn}"
+ request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
+ control_plane_base: Final = bedrock_control_plane_base_url(
+ request_params.get("aws_bedrock_control_plane_endpoint"), region
)
+ endpoint_url: Final = f"{control_plane_base}/model-invocation-job/{encoded_arn}"
# Use common utility for AWS signing
- request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
signed_headers, _ = self.common_utils.sign_aws_request(
service_name="bedrock",
data={}, # GET request has no body
diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py
index bbbc929fad3..0cf3e895ad2 100644
--- a/litellm/llms/bedrock/common_utils.py
+++ b/litellm/llms/bedrock/common_utils.py
@@ -138,6 +138,25 @@ def merge_bedrock_aws_request_params(
return request_params
+def configured_bedrock_control_plane_endpoint(endpoint: object) -> str | None:
+ """An ``aws_bedrock_control_plane_endpoint`` value without a trailing slash, or None when unset."""
+ if isinstance(endpoint, str) and endpoint.strip():
+ return endpoint.strip().rstrip("/")
+ return None
+
+
+def bedrock_control_plane_base_url(endpoint: object, aws_region_name: str) -> str:
+ """Base URL for Bedrock control-plane calls such as model invocation jobs.
+
+ A configured ``aws_bedrock_control_plane_endpoint`` (a VPC endpoint or a compatible gateway)
+ wins over the regional ``bedrock`` host. ``api_base`` is deliberately not used here, since on a
+ Bedrock deployment it points at the runtime host, which does not serve these APIs.
+ """
+ return configured_bedrock_control_plane_endpoint(endpoint) or (
+ f"https://bedrock.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}"
+ )
+
+
def s3_static_key_pair(params: Mapping[str, object]) -> tuple[str, str] | None:
"""The s3_access_key_id / s3_secret_access_key pair when both are set, otherwise None."""
s3_access_key_id: Final = params.get("s3_access_key_id")
diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index 73d20ae632c..996c28bebaa 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -419,6 +419,7 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = (
# tokens) to the attacker's host, or coerces the proxy into
# authenticating against the attacker's host with admin secrets.
"aws_bedrock_runtime_endpoint",
+ "aws_bedrock_control_plane_endpoint",
# Bedrock project/workspace association. Deployments pin this to
# enforce a data-retention policy, so a caller-supplied value would
# re-route the request's retention and accounting to any project
diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py
index 978eb9c4557..6b08920e042 100644
--- a/litellm/types/litellm_params.py
+++ b/litellm/types/litellm_params.py
@@ -103,6 +103,7 @@ class ProviderConnection:
class BedrockBatchConnection:
# Bedrock rejects these names in request bodies, so register them as LiteLLM-owned
aws_batch_role_arn: str | None = None
+ aws_bedrock_control_plane_endpoint: str | None = None
s3_bucket_name: str | None = None
s3_region_name: str | None = None
s3_endpoint_url: str | None = None
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 1f159cb8b16..2618bf698ee 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -413,6 +413,7 @@ class CredentialLiteLLMParams(LiteLLMBaseModel):
aws_external_id: str | None = None
aws_session_tags: Sequence[AwsSessionTag] | None = None
aws_bedrock_runtime_endpoint: str | None = None
+ aws_bedrock_control_plane_endpoint: str | None = None
aws_bedrock_project_id: str | None = None
s3_bucket_name: str | None = None
s3_endpoint_url: str | None = None
diff --git a/tests/unit/llms/bedrock/batches/test_handler.py b/tests/unit/llms/bedrock/batches/test_handler.py
index d920eabcdf6..602504f0c49 100644
--- a/tests/unit/llms/bedrock/batches/test_handler.py
+++ b/tests/unit/llms/bedrock/batches/test_handler.py
@@ -257,6 +257,25 @@ def test_explicit_region_overrides_arn(patched_boto3):
assert kwargs["region_name"] == "eu-central-1"
+def test_status_client_uses_regional_host_when_no_control_plane_endpoint(
+ patched_boto3: tuple[MagicMock, MagicMock],
+) -> None:
+ _, boto_client_factory = patched_boto3
+ BedrockBatchesHandler.handle_model_invocation_job_status(batch_id=JOB_ARN)
+ _, kwargs = boto_client_factory.call_args
+ assert kwargs["endpoint_url"] is None
+
+
+def test_status_client_uses_configured_control_plane_endpoint(patched_boto3: tuple[MagicMock, MagicMock]) -> None:
+ _, boto_client_factory = patched_boto3
+ BedrockBatchesHandler.handle_model_invocation_job_status(
+ batch_id=JOB_ARN,
+ aws_bedrock_control_plane_endpoint="https://bedrock-gateway.internal/",
+ )
+ _, kwargs = boto_client_factory.call_args
+ assert kwargs["endpoint_url"] == "https://bedrock-gateway.internal"
+
+
def test_failure_message_propagates(patched_boto3):
fake_client, _ = patched_boto3
failed_response = _fake_boto3_response(status="Failed")
@@ -479,6 +498,29 @@ def test_litellm_cancel_batch_dispatches_to_bedrock(patched_boto3):
assert batch.status == "cancelled"
+def test_litellm_batch_calls_send_every_bedrock_client_to_the_configured_endpoint(
+ patched_boto3: tuple[MagicMock, MagicMock],
+) -> None:
+ import litellm
+
+ fake_client, boto_client_factory = patched_boto3
+ fake_client.get_model_invocation_job.return_value = _fake_boto3_response(status="Stopped")
+
+ litellm.cancel_batch(
+ batch_id=JOB_ARN,
+ custom_llm_provider="bedrock",
+ aws_bedrock_control_plane_endpoint="https://bedrock-gateway.internal",
+ )
+ litellm.retrieve_batch(
+ batch_id=JOB_ARN,
+ custom_llm_provider="bedrock",
+ aws_bedrock_control_plane_endpoint="https://bedrock-gateway.internal",
+ )
+
+ endpoints: Final = tuple(call.kwargs["endpoint_url"] for call in boto_client_factory.call_args_list)
+ assert endpoints == ("https://bedrock-gateway.internal",) * 3
+
+
class _TagGatedSTSClient:
"""Stands in for STS behind a trust policy that only admits sessions carrying ``tags``."""
diff --git a/tests/unit/llms/bedrock/batches/test_transformation.py b/tests/unit/llms/bedrock/batches/test_transformation.py
index 5e987239c54..5360ac00018 100644
--- a/tests/unit/llms/bedrock/batches/test_transformation.py
+++ b/tests/unit/llms/bedrock/batches/test_transformation.py
@@ -14,10 +14,15 @@ URL/ARN handling, and the error class. AWS auth/sigv4 is the only external seam
we mock; everything else runs for real.
"""
+from datetime import datetime, timezone
+from typing import Final
from unittest.mock import MagicMock, patch
import httpx
import pytest
+from botocore.auth import SigV4Auth
+from botocore.awsrequest import AWSRequest
+from botocore.credentials import Credentials
from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig
from litellm.types.utils import LlmProviders
@@ -41,7 +46,7 @@ ARN = "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/abc1234567"
@pytest.fixture
-def config():
+def config() -> BedrockBatchesConfig:
return BedrockBatchesConfig()
@@ -66,6 +71,90 @@ def test_get_complete_batch_url_uses_region(config):
assert url == "https://bedrock.eu-central-1.amazonaws.com/model-invocation-job"
+CONTROL_PLANE_ENDPOINT: Final = "https://vpce-0abc.bedrock.us-west-2.vpce.amazonaws.com/"
+
+
+def test_get_complete_batch_url_uses_configured_control_plane_endpoint(config: BedrockBatchesConfig) -> None:
+ url: Final = config.get_complete_batch_url(
+ api_base=None,
+ api_key=None,
+ model="anthropic.claude-3",
+ optional_params={"aws_region_name": "us-west-2"},
+ litellm_params={"aws_bedrock_control_plane_endpoint": CONTROL_PLANE_ENDPOINT},
+ data={"input_file_id": "s3://b/k"},
+ )
+ assert url == "https://vpce-0abc.bedrock.us-west-2.vpce.amazonaws.com/model-invocation-job"
+
+
+def test_proxy_deployment_credentials_carry_the_control_plane_endpoint(config: BedrockBatchesConfig) -> None:
+ from litellm import Router
+
+ router: Final = Router(
+ model_list=[
+ {
+ "model_name": "claude-batch",
+ "litellm_params": {
+ "model": "bedrock/anthropic.claude-3",
+ "aws_region_name": "us-west-2",
+ "aws_bedrock_control_plane_endpoint": CONTROL_PLANE_ENDPOINT,
+ },
+ }
+ ]
+ )
+ credentials: Final = router.get_deployment_credentials_with_provider("claude-batch")
+
+ url: Final = config.get_complete_batch_url(
+ api_base=None,
+ api_key=None,
+ model="anthropic.claude-3",
+ optional_params={},
+ litellm_params=credentials,
+ data={"input_file_id": "s3://b/k"},
+ )
+ assert url == "https://vpce-0abc.bedrock.us-west-2.vpce.amazonaws.com/model-invocation-job"
+
+
+def test_get_complete_batch_url_ignores_runtime_api_base(config: BedrockBatchesConfig) -> None:
+ url: Final = config.get_complete_batch_url(
+ api_base="https://bedrock-runtime.us-west-2.amazonaws.com",
+ api_key=None,
+ model="anthropic.claude-3",
+ optional_params={"aws_region_name": "us-west-2"},
+ litellm_params={"api_base": "https://bedrock-runtime.us-west-2.amazonaws.com"},
+ data={"input_file_id": "s3://b/k"},
+ )
+ assert url == "https://bedrock.us-west-2.amazonaws.com/model-invocation-job"
+
+
+def test_create_request_sends_and_signs_for_configured_control_plane_endpoint(config: BedrockBatchesConfig) -> None:
+ expected_url: Final = "https://vpce-0abc.bedrock.us-west-2.vpce.amazonaws.com/model-invocation-job"
+ with patch("botocore.auth.get_current_datetime", return_value=datetime(2026, 1, 1, tzinfo=timezone.utc)):
+ result: Final = config.transform_create_batch_request(
+ model="anthropic.claude-haiku-4-5-20251001-v1:0",
+ create_batch_data={
+ "input_file_id": "s3://in-bucket/input.jsonl",
+ "endpoint": "/v1/chat/completions",
+ "completion_window": "24h",
+ },
+ optional_params={"aws_region_name": "us-west-2"},
+ litellm_params={
+ "aws_batch_role_arn": "arn:aws:iam::123:role/my-batch-role",
+ "aws_bedrock_control_plane_endpoint": CONTROL_PLANE_ENDPOINT,
+ "aws_access_key_id": "endpoint-test-key",
+ "aws_secret_access_key": "endpoint-test-secret",
+ },
+ )
+ expected_request: Final = AWSRequest(
+ method="POST", url=expected_url, data=result["data"], headers={"Content-Type": "application/json"}
+ )
+ SigV4Auth(Credentials("endpoint-test-key", "endpoint-test-secret"), "bedrock", "us-west-2").add_auth(
+ expected_request
+ )
+
+ assert result["url"] == expected_url
+ assert result["headers"]["Authorization"] == expected_request.headers["Authorization"]
+
+
# --------------------------------------------------------------------------- #
# transform_create_batch_request - request construction (sign_aws_request mocked)
# --------------------------------------------------------------------------- #
@@ -608,6 +697,30 @@ def test_retrieve_request_builds_encoded_arn_url(config):
assert mock_sign.call_args.kwargs["data"] == {}
+def test_retrieve_request_sends_and_signs_for_configured_control_plane_endpoint(config: BedrockBatchesConfig) -> None:
+ expected_url: Final = (
+ "https://vpce-0abc.bedrock.us-west-2.vpce.amazonaws.com/model-invocation-job/"
+ "arn%3Aaws%3Abedrock%3Aus-west-2%3A123456789012%3Amodel-invocation-job%2Fabc1234567"
+ )
+ with patch("botocore.auth.get_current_datetime", return_value=datetime(2026, 1, 1, tzinfo=timezone.utc)):
+ result: Final = config.transform_retrieve_batch_request(
+ batch_id=ARN,
+ optional_params={},
+ litellm_params={
+ "aws_bedrock_control_plane_endpoint": CONTROL_PLANE_ENDPOINT,
+ "aws_access_key_id": "endpoint-test-key",
+ "aws_secret_access_key": "endpoint-test-secret",
+ },
+ )
+ expected_request: Final = AWSRequest(method="GET", url=expected_url, data="", headers={})
+ SigV4Auth(Credentials("endpoint-test-key", "endpoint-test-secret"), "bedrock", "us-west-2").add_auth(
+ expected_request
+ )
+
+ assert result["url"] == expected_url
+ assert result["headers"]["Authorization"] == expected_request.headers["Authorization"]
+
+
def test_retrieve_request_rejects_non_arn(config):
with pytest.raises(ValueError, match="Expected ARN"):
config.transform_retrieve_batch_request(
diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py
index 1fb5542c964..154797bab5a 100644
--- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py
+++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py
@@ -10739,3 +10739,53 @@ def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_
assert route.call_count == expected_calls
assert all("outputConfig" in json.loads(call.request.content) for call in route.calls[:1])
assert all("outputConfig" not in json.loads(call.request.content) for call in route.calls[1:])
+
+
+CONTROL_PLANE_ENDPOINT: Final = "https://bedrock-fips.us-east-1.amazonaws.com"
+
+
+def _bedrock_chat_answer(request: httpx.Request) -> httpx.Response:
+ if request.url.path.endswith("/invoke"):
+ return httpx.Response(
+ 200,
+ json={
+ "id": "msg_control_plane",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-haiku-4-5",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 1, "output_tokens": 1},
+ },
+ )
+ return httpx.Response(
+ 200,
+ json={
+ "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
+ "stopReason": "end_turn",
+ "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
+ },
+ )
+
+
+@pytest.mark.parametrize("route_prefix", ["bedrock/", "bedrock/converse/", "bedrock/invoke/"])
+@pytest.mark.respx(assert_all_called=True)
+def test_a_deployment_with_a_batch_control_plane_endpoint_keeps_it_out_of_chat_requests(
+ respx_mock: respx.Router, route_prefix: str
+) -> None:
+ route: Final = respx_mock.post(
+ url__regex=r"^https://bedrock-runtime\.us-east-1\.amazonaws\.com/model/.+/(converse|invoke)$"
+ ).mock(side_effect=_bedrock_chat_answer)
+
+ response: Final = litellm.completion(
+ model=f"{route_prefix}us.anthropic.claude-haiku-4-5-20251001-v1:0",
+ messages=[{"role": "user", "content": "hi"}],
+ aws_access_key_id="AKIACONTROLPLANE",
+ aws_secret_access_key="control-plane-secret",
+ aws_region_name="us-east-1",
+ aws_bedrock_control_plane_endpoint=CONTROL_PLANE_ENDPOINT,
+ client=HTTPHandler(),
+ )
+
+ assert response.choices[0].message.content == "ok"
+ assert "aws_bedrock_control_plane_endpoint" not in route.calls[0].request.content.decode()
diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py
index f81ba155097..67ff7dd9344 100644
--- a/tests/unit/proxy/auth/test_auth_utils.py
+++ b/tests/unit/proxy/auth/test_auth_utils.py
@@ -3566,6 +3566,7 @@ class TestIsRequestBodySafeBlocksEndpointTargetingFields:
"field",
[
"aws_bedrock_runtime_endpoint",
+ "aws_bedrock_control_plane_endpoint",
"langsmith_base_url",
"langfuse_host",
"posthog_host",
diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py
index e88d757519c..84ba85f4a68 100644
--- a/tests/unit/types/test_litellm_params.py
+++ b/tests/unit/types/test_litellm_params.py
@@ -95,6 +95,7 @@ CONNECTION_NAMES: Final = (
"use_xai_oauth",
"fireworks_forward_user_id",
"aws_batch_role_arn",
+ "aws_bedrock_control_plane_endpoint",
"s3_bucket_name",
"s3_region_name",
"s3_endpoint_url",
@@ -274,6 +275,7 @@ INTERNAL_STATE_NAMES: Final = (
BEDROCK_BATCH_NAMES: Final = (
"aws_batch_role_arn",
+ "aws_bedrock_control_plane_endpoint",
"s3_bucket_name",
"s3_region_name",
"s3_endpoint_url",
@@ -686,6 +688,7 @@ NAMES_SHARED_WITH_TYPED_MODELS: Final[Mapping[str, tuple[str, ...]]] = MappingPr
"api_key",
"api_version",
"aws_batch_role_arn",
+ "aws_bedrock_control_plane_endpoint",
"azure_password",
"azure_scope",
"azure_username",
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index b93f7dce2ca..e432c947116 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -35048,6 +35048,8 @@ export interface components {
aws_access_key_id?: string | null;
/** Aws Batch Role Arn */
aws_batch_role_arn?: string | null;
+ /** Aws Bedrock Control Plane Endpoint */
+ aws_bedrock_control_plane_endpoint?: string | null;
/** Aws Bedrock Project Id */
aws_bedrock_project_id?: string | null;
/** Aws Bedrock Runtime Endpoint */
@@ -50336,6 +50338,8 @@ export interface components {
aws_access_key_id?: string | null;
/** Aws Batch Role Arn */
aws_batch_role_arn?: string | null;
+ /** Aws Bedrock Control Plane Endpoint */
+ aws_bedrock_control_plane_endpoint?: string | null;
/** Aws Bedrock Project Id */
aws_bedrock_project_id?: string | null;
/** Aws Bedrock Runtime Endpoint */
From 767b75aa9db618e29e43075917d29a84c2aab23e Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 12:31:00 -0700
Subject: [PATCH 026/203] test(perf): record the data request status per UI
journey sample and bound each journey with a wall clock (#47622)
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
tests/e2e/perf/UI_QUERIES.md | 2 +
tests/e2e/perf/setup/ui_journeys.py | 37 +++++++++++++++++--
tests/e2e/perf/test_ui_journeys.py | 30 ++++++++++++++-
.../perf/setup/test_ui_journeys.py | 11 ++++++
4 files changed, 76 insertions(+), 4 deletions(-)
diff --git a/tests/e2e/perf/UI_QUERIES.md b/tests/e2e/perf/UI_QUERIES.md
index 27780537042..9bf82c8c07e 100644
--- a/tests/e2e/perf/UI_QUERIES.md
+++ b/tests/e2e/perf/UI_QUERIES.md
@@ -53,3 +53,5 @@ uv run pytest tests/e2e/perf/test_ui_journeys.py --reruns 0 --junitxml="$E2E_ART
Each JUnit testcase `test_ui_journey_latency[]` carries `perf_ui_journeys`, `perf_ui_failures`, `perf_ui_seconds_p50`, `perf_ui_seconds_p90`, `perf_ui_seconds_max`, `perf_ui_first_seconds`, `perf_ui_repeated_seconds_p50`, `perf_ui_repeated_seconds_max` and `perf_ui_database_failures`. `ui-journey-.json` holds every sample with its on-screen row count and `ui-isolation-.jsonl` the SQL isolation evidence
The Buildkite `ui_queries` lane restores the pinned snapshot, waits for migrations and indexes, runs these journeys with no retries and uploads the JUnit and JSON artifacts. Validate selectors on a small restore first, then take numbers from the completed 1TB snapshot
+
+Each sample in `ui-journey-.json` also records the `status` and `request` (method and path) of the data request the page issued, the first non-2xx one when there is one, so a failed sample names what the gateway answered. A failed sample with a 4xx or 5xx status is counted as `perf_ui_error_http_` instead of `perf_ui_error_ui`. Each journey is also bounded by a wall clock of `E2E_UI_JOURNEY_TIMEOUT_SECONDS x (repeats + 2)`, so one hung browser call ends the journey with a `wall_clock` sample instead of eating the lane.
diff --git a/tests/e2e/perf/setup/ui_journeys.py b/tests/e2e/perf/setup/ui_journeys.py
index e9d4360d1a4..2b7bda77733 100644
--- a/tests/e2e/perf/setup/ui_journeys.py
+++ b/tests/e2e/perf/setup/ui_journeys.py
@@ -10,6 +10,7 @@ from dataclasses import dataclass
from datetime import datetime
from itertools import chain
from types import MappingProxyType
+from urllib.parse import urlsplit
from typing import Final
from playwright.sync_api import Error as PlaywrightError
@@ -30,6 +31,8 @@ class Sample(BaseModel):
seconds: float
rows: int = 0
error: str = ""
+ status: int = 0
+ request: str = ""
class Timings(BaseModel):
@@ -80,6 +83,10 @@ def summarize(samples: tuple[Sample, ...]) -> Timings:
def error_code(sample: Sample) -> str:
if not sample.error:
return ""
+ if sample.status >= 400:
+ return f"http_{sample.status}"
+ if sample.error.startswith("wall_clock"):
+ return "wall_clock"
return "timeout" if sample.error == "timeout" else "ui"
@@ -185,6 +192,19 @@ def _is_data_fetch(response: Response) -> bool:
return response.request.resource_type in ("fetch", "xhr") and "/_next/" not in response.url
+def _worst(responses: list[Response]) -> Response | None:
+ return next((response for response in responses if response.status >= 400), responses[-1] if responses else None)
+
+
+def _with_response(sample: Sample, responses: list[Response]) -> Sample:
+ worst: Final = _worst(responses)
+ if worst is None:
+ return sample
+ return sample.model_copy(
+ update={"status": worst.status, "request": f"{worst.request.method} {urlsplit(worst.url).path}"[:120]}
+ )
+
+
def measure(page: Page, journey: Journey, timeout: float) -> Sample:
"""Wall time from the user's action to the data being on screen. Every sample starts from the
same pre-action state so repeats measure the same journey, not a warmed client cache."""
@@ -192,16 +212,27 @@ def measure(page: Page, journey: Journey, timeout: float) -> Sample:
journey.start(page, timeout)
except (PlaywrightError, AssertionError) as error:
return Sample(seconds=0.0, error=f"start {type(error).__name__}: {error}"[:300])
+ responses: Final[list[Response]] = [] # mutable-ok: filled by the playwright response listener
+
+ def record(response: Response) -> None:
+ if _is_data_fetch(response):
+ responses.append(response)
+
+ page.on("response", record)
started: Final = time.monotonic()
try:
with page.expect_response(_is_data_fetch, timeout=timeout * 1000):
journey.act(page)
rows: Final = journey.settled(page, timeout)
except PlaywrightTimeoutError:
- return Sample(seconds=time.monotonic() - started, error="timeout")
+ return _with_response(Sample(seconds=time.monotonic() - started, error="timeout"), responses)
except (PlaywrightError, AssertionError) as error:
- return Sample(seconds=time.monotonic() - started, error=f"{type(error).__name__}: {error}"[:300])
- return Sample(seconds=time.monotonic() - started, rows=rows)
+ return _with_response(
+ Sample(seconds=time.monotonic() - started, error=f"{type(error).__name__}: {error}"[:300]), responses
+ )
+ finally:
+ page.remove_listener("response", record)
+ return _with_response(Sample(seconds=time.monotonic() - started, rows=rows), responses)
def _visible(page: Page, test_id: str) -> Locator:
diff --git a/tests/e2e/perf/test_ui_journeys.py b/tests/e2e/perf/test_ui_journeys.py
index 888a44d2e27..db25608a00d 100644
--- a/tests/e2e/perf/test_ui_journeys.py
+++ b/tests/e2e/perf/test_ui_journeys.py
@@ -2,11 +2,16 @@ from __future__ import annotations
import json
import os
+import signal
+import time
+from collections.abc import Generator
from datetime import UTC, datetime
from pathlib import Path
+from types import FrameType
from typing import Final
import pytest
+from playwright.sync_api import Page
from e2e_config import MASTER_KEY, UI_BASE_URL, UI_PASSWORD, UI_USERNAME
from e2e_http import send_text_external, without_retries
from setup.ui_journeys import (
@@ -56,7 +61,30 @@ def _metrics(transport: HttpTransport) -> float:
def _samples(dashboard: Dashboard, journey: Journey, timeout: float, repeats: int) -> tuple[Sample, ...]:
with dashboard_page(dashboard, timeout) as page:
- return tuple(measure(page, journey, timeout) for _ in range(repeats + 1))
+ return tuple(_within_budget(page, journey, timeout, repeats))
+
+
+def _within_budget(page: Page, journey: Journey, timeout: float, repeats: int) -> Generator[Sample]:
+ """One hanging browser call must not eat the lane: SIGALRM ends the journey at timeout x (repeats + 2),
+ the budget every sample needs (start plus act plus settle) with one spare sample of slack."""
+ budget: Final = timeout * (repeats + 2)
+
+ def expire(signum: int, frame: FrameType | None) -> None:
+ raise TimeoutError(f"{journey.name} exceeded its {budget:.0f}s wall clock")
+
+ previous: Final = signal.signal(signal.SIGALRM, expire)
+ started: Final = time.monotonic()
+ try:
+ _ = signal.setitimer(signal.ITIMER_REAL, budget)
+ for _ in range(repeats + 1):
+ yield measure(page, journey, timeout)
+ except Exception as error:
+ if time.monotonic() - started < budget:
+ raise
+ yield Sample(seconds=time.monotonic() - started, error=f"wall_clock {type(error).__name__}: {error}"[:300])
+ finally:
+ _ = signal.setitimer(signal.ITIMER_REAL, 0)
+ _ = signal.signal(signal.SIGALRM, previous)
def test_ui_journey_latency(journey_name: str, ui_fixture: UiFixture, request: pytest.FixtureRequest) -> None:
diff --git a/tests/e2e_harness/perf/setup/test_ui_journeys.py b/tests/e2e_harness/perf/setup/test_ui_journeys.py
index 7d5b0d531ec..dd2c1fc0301 100644
--- a/tests/e2e_harness/perf/setup/test_ui_journeys.py
+++ b/tests/e2e_harness/perf/setup/test_ui_journeys.py
@@ -6,6 +6,7 @@ import pytest
from setup.ui_journeys import (
UI_JOURNEY_LIMITS,
Sample,
+ error_code,
assert_within_ceiling,
database_failures,
journeys,
@@ -142,3 +143,13 @@ def test_slow_journey_fails_its_ceiling() -> None:
assert_within_ceiling("logs_open_24h", (*fast[:8], Sample(seconds=25, rows=25), Sample(seconds=25, rows=25)))
with pytest.raises(AssertionError, match=r"logs_open_24h p50 11\.0s over 10s"):
assert_within_ceiling("logs_open_24h", tuple(Sample(seconds=11, rows=25) for _ in range(3)))
+
+
+def test_failed_data_request_status_becomes_the_error_code() -> None:
+ gateway_error: Final = Sample(seconds=0.4, error="AssertionError: Locator expected to be enabled", status=500)
+ slow: Final = Sample(seconds=60.0, error="timeout", status=200, request="GET /spend/logs/ui")
+ hung: Final = Sample(seconds=700.0, error="wall_clock TimeoutError: logs_next_page exceeded its 660s wall clock")
+ assert (error_code(gateway_error), error_code(slow), error_code(hung)) == ("http_500", "timeout", "wall_clock")
+ metrics: Final = dict(ui_junit_properties((gateway_error, slow, hung)))
+ assert metrics["perf_ui_error_http_500"] == 1 and metrics["perf_ui_error_wall_clock"] == 1
+ assert "perf_ui_error_ui" not in metrics
From 280496b9dd2dbb7245ed3a5da22477074f7b518b Mon Sep 17 00:00:00 2001
From: Kerry Lu
Date: Tue, 20 Oct 2026 12:49:58 -0700
Subject: [PATCH 027/203] test(perf): let the UI journey fixture run on a
restore upgraded by a released migrations image (#47628)
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
tests/e2e/perf/UI_QUERIES.md | 2 ++
tests/e2e/perf/setup/ui_query_fixture.py | 9 +++++++++
2 files changed, 11 insertions(+)
diff --git a/tests/e2e/perf/UI_QUERIES.md b/tests/e2e/perf/UI_QUERIES.md
index 9bf82c8c07e..d2cbcd368fc 100644
--- a/tests/e2e/perf/UI_QUERIES.md
+++ b/tests/e2e/perf/UI_QUERIES.md
@@ -54,4 +54,6 @@ Each JUnit testcase `test_ui_journey_latency[]` carries `perf_ui_journeys`
The Buildkite `ui_queries` lane restores the pinned snapshot, waits for migrations and indexes, runs these journeys with no retries and uploads the JUnit and JSON artifacts. Validate selectors on a small restore first, then take numbers from the completed 1TB snapshot
+To measure a released version on the same snapshot, create the litellm-perf build with `perf_gateway_version=` (for example `v1.106.1`) next to `litellm_sha`. The gateway, backend and the restore's schema upgrade then come from `ghcr.io/berriai/litellm-{gateway,backend,migrations}:`, where the migrations image runs `prisma migrate deploy` and builds the request-log indexes the way a customer upgrade does, while the runner and these journeys come from `litellm_sha`. The fixture skips its upgrade-worker wait when `_LiteLLMPerfUpgrade` is absent. The snapshot's schema is v1.105.1 and the restore only migrates forward, so older tags cannot be measured
+
Each sample in `ui-journey-.json` also records the `status` and `request` (method and path) of the data request the page issued, the first non-2xx one when there is one, so a failed sample names what the gateway answered. A failed sample with a 4xx or 5xx status is counted as `perf_ui_error_http_` instead of `perf_ui_error_ui`. Each journey is also bounded by a wall clock of `E2E_UI_JOURNEY_TIMEOUT_SECONDS x (repeats + 2)`, so one hung browser call ends the journey with a `wall_clock` sample instead of eating the lane.
diff --git a/tests/e2e/perf/setup/ui_query_fixture.py b/tests/e2e/perf/setup/ui_query_fixture.py
index 1cbb5c41ea3..cfffa0822e8 100644
--- a/tests/e2e/perf/setup/ui_query_fixture.py
+++ b/tests/e2e/perf/setup/ui_query_fixture.py
@@ -105,10 +105,19 @@ def _count(row: tuple[object, ...] | None) -> int:
return TypeAdapter(int).validate_python(row[0])
+def _released_schema(connection: psycopg.Connection) -> bool:
+ """A released tag's migrations image upgrades the restore (schema and request-log indexes) before the
+ stack starts, so there is no upgrade worker state to wait on."""
+ row: Final = connection.execute("""SELECT to_regclass('public."_LiteLLMPerfUpgrade"') IS NULL""").fetchone()
+ return row is None or TypeAdapter(bool).validate_python(row[0])
+
+
def _finish_upgrade() -> None:
if os.environ.get("E2E_DATABASE_UPGRADE") != "1":
return
with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True, connect_timeout=10) as connection:
+ if _released_schema(connection):
+ return
release_indexes(connection)
deadline: Final = time.monotonic() + 6 * 3600
while time.monotonic() < deadline:
From c20d57c18ad30022b682b5586e8b990e1e1271ec Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 20:02:29 +0000
Subject: [PATCH 028/203] fix(ui): rebuild the dashboard bundle and cover it in
the example master key check (#47612)
* chore: update Next.js build artifacts (2026-10-20 18:15 UTC, node v24.19.0)
* test(security): cover the generated dashboard bundle in the example master key check
* test(security): keep a clean-repo success case for the example master key check
---
litellm/proxy/_experimental/out/404.html | 2 +-
.../proxy/_experimental/out/404/index.html | 2 +-
.../out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt | 39 +-
.../proxy/_experimental/out/__next._full.txt | 64 +--
.../proxy/_experimental/out/__next._tree.txt | 6 +-
.../out/_next/static/chunks/0-p31wd20yje5.js | 1 +
.../out/_next/static/chunks/0-w0rr3df0htc.js | 1 -
.../out/_next/static/chunks/0-xrvklzr286n.js | 1 +
.../out/_next/static/chunks/006y36jxl8z-u.js | 1 -
.../out/_next/static/chunks/01cjt3b43zsyy.js | 23 +
.../out/_next/static/chunks/01f03qwhd6l7d.js | 1 -
.../out/_next/static/chunks/01msi0ehnlc7g.js | 1 +
.../out/_next/static/chunks/02erlqrgtvdvz.js | 1 -
.../out/_next/static/chunks/02mlplp0iptro.js | 7 -
.../out/_next/static/chunks/033urjy22ackz.js | 1 -
.../out/_next/static/chunks/034i32t0-7tdv.js | 1 -
.../out/_next/static/chunks/03kp2as6u_s_6.js | 1 +
.../out/_next/static/chunks/03xf0a_nt1mqx.js | 1 -
.../out/_next/static/chunks/04m1lhogzlu_q.js | 1 -
.../out/_next/static/chunks/058x5ogyudznz.js | 1 -
.../out/_next/static/chunks/05cei057fcv33.js | 1 +
.../out/_next/static/chunks/05evi9aern8_5.js | 1 +
.../out/_next/static/chunks/05p3egzpg0fru.js | 434 +++++++++++++++++
.../out/_next/static/chunks/05php4kcqbp33.js | 1 -
.../out/_next/static/chunks/0677cowehlx_9.js | 1 +
.../out/_next/static/chunks/06el5-9ee247i.js | 60 +++
.../out/_next/static/chunks/06yuul4yvgbd0.css | 1 +
.../out/_next/static/chunks/07ifwp0wdqloz.js | 1 +
.../out/_next/static/chunks/07xbdb1bjx9eg.js | 1 -
.../out/_next/static/chunks/08x62apceyttf.js | 3 +
.../out/_next/static/chunks/08x8fsl2hzbmq.js | 1 +
.../out/_next/static/chunks/095t3bxug2-v_.js | 3 +
.../out/_next/static/chunks/09kfqcp7rqvl7.js | 1 -
.../out/_next/static/chunks/09pcs5yy22ada.js | 1 -
.../out/_next/static/chunks/09qlj1_ya5uqw.js | 1 -
.../out/_next/static/chunks/09tfr__sbh796.js | 1 +
.../out/_next/static/chunks/0_50gw2o6ba0j.js | 34 ++
.../out/_next/static/chunks/0_8vcd9i7eo1r.js | 1 -
.../out/_next/static/chunks/0__ufucx2g6ui.js | 1 -
.../out/_next/static/chunks/0_t1_1-2to_0w.js | 1 -
.../out/_next/static/chunks/0ao344k1l0l2h.js | 23 -
.../out/_next/static/chunks/0bhhnflodjdhp.js | 1 +
.../out/_next/static/chunks/0bks94633rs4s.js | 1 -
.../out/_next/static/chunks/0bl98f8jm71jn.js | 35 ++
.../out/_next/static/chunks/0blpz77qmgyho.js | 1 +
.../out/_next/static/chunks/0ca8wd799xeut.js | 1 +
.../{1kid9zr1--h6y.css => 0ckv5j_fni3xm.css} | 2 +-
.../out/_next/static/chunks/0d-myyz3fnj3y.js | 213 +++++++++
.../out/_next/static/chunks/0dyoztesepp-8.js | 1 -
.../out/_next/static/chunks/0ec5zmg5_3qwx.js | 1 -
.../out/_next/static/chunks/0eclgobdwjfbw.js | 1 +
.../out/_next/static/chunks/0eh6yvm8qswse.js | 1 -
.../out/_next/static/chunks/0el1dng5997zf.js | 1 +
.../out/_next/static/chunks/0eruvlogn4q8l.js | 4 +
.../out/_next/static/chunks/0ev0y17rt0bzh.js | 1 +
.../out/_next/static/chunks/0eybcbrej9bl8.js | 1 -
.../out/_next/static/chunks/0f4daktg2mnzj.js | 1 +
.../out/_next/static/chunks/0f74g4_7j-4ju.js | 1 +
.../out/_next/static/chunks/0gjxeo9f--vzt.js | 1 +
.../out/_next/static/chunks/0h6b6ooi-yfmn.js | 1 -
.../out/_next/static/chunks/0hukc5zl2txhg.js | 1 +
.../out/_next/static/chunks/0i--ursme41qh.js | 1 -
.../out/_next/static/chunks/0iad0mpbdl5k9.js | 1 +
.../out/_next/static/chunks/0icwvr3xvxvlx.js | 1 +
.../out/_next/static/chunks/0j0877r_j866g.js | 1 +
.../{2atns3zer6s42.js => 0jau1crjcgz3m.js} | 6 +-
.../out/_next/static/chunks/0ldd7ocximwhh.js | 1 -
.../out/_next/static/chunks/0md57zg_zhxqq.js | 1 -
.../out/_next/static/chunks/0n3xjld_n2chp.js | 5 -
.../out/_next/static/chunks/0nqgb35_9l922.js | 16 +
.../out/_next/static/chunks/0o2yjrr5g_tdw.js | 1 +
.../out/_next/static/chunks/0och7iysqbm0w.js | 1 +
.../out/_next/static/chunks/0pz3k7bzm5al8.js | 1 -
.../out/_next/static/chunks/0q6x33fc-tn6m.js | 1 +
.../out/_next/static/chunks/0qlyu_3ohy0_9.js | 1 -
.../out/_next/static/chunks/0r0nhtsbxio43.js | 1 -
.../out/_next/static/chunks/0r2no56zz5i7e.js | 1 -
.../out/_next/static/chunks/0refy__7v6-qb.js | 1 +
.../out/_next/static/chunks/0rq646fx4-bql.js | 1 -
.../out/_next/static/chunks/0sn6ne06gs8iu.js | 421 -----------------
.../out/_next/static/chunks/0stffhbqahki3.js | 1 -
.../{0_u7rrnrqu95t.js => 0svv6cnxs4pbi.js} | 2 +-
.../out/_next/static/chunks/0sz89fsnzc09a.js | 1 -
.../out/_next/static/chunks/0t3a_qboss-93.js | 1 -
.../out/_next/static/chunks/0tzl5rama7x4_.js | 1 -
.../out/_next/static/chunks/0ucu5tumma10r.js | 2 +
.../out/_next/static/chunks/0ui61y5hgz0ck.js | 1 -
.../out/_next/static/chunks/0uo3fyy_3adzw.js | 1 -
.../out/_next/static/chunks/0vpn3th7sn4vf.js | 1 -
.../out/_next/static/chunks/0w5e2qyxu1sj4.js | 1 +
.../out/_next/static/chunks/0wk0l0tbb11zh.js | 5 +
.../out/_next/static/chunks/0x88jgebq4fjq.js | 4 -
.../out/_next/static/chunks/0xfewjecn0ixz.js | 1 +
.../out/_next/static/chunks/0xl3regan_n7s.js | 1 -
.../out/_next/static/chunks/0xsuy-8_q50ub.js | 68 ---
.../out/_next/static/chunks/0ye1nryroojck.js | 1 +
.../out/_next/static/chunks/0zi1szq8x_gmi.js | 1 +
.../out/_next/static/chunks/0zm9m2grnntd3.js | 1 +
.../out/_next/static/chunks/1-metsezi443m.js | 31 --
.../out/_next/static/chunks/1-zxupj2b23jk.js | 7 +
.../out/_next/static/chunks/10i7gzq_kbeak.js | 1 +
.../out/_next/static/chunks/115x4nuphlkvv.js | 1 -
.../out/_next/static/chunks/11c-g4jel910l.js | 1 -
.../out/_next/static/chunks/11eb6uxtl-k7c.js | 56 ---
.../out/_next/static/chunks/11o_e34ji0wx-.js | 1 -
.../out/_next/static/chunks/11vytukfj5_7x.js | 1 -
.../out/_next/static/chunks/12_-wfvirgolu.js | 1 -
.../out/_next/static/chunks/12thj6aa-tc5c.js | 1 +
.../{0lwia0t_dwgb-.js => 13douredghzbr.js} | 2 +-
.../out/_next/static/chunks/13lpebw2fp-wr.js | 216 +++++++++
.../out/_next/static/chunks/140374h5u57ha.js | 1 +
.../out/_next/static/chunks/142sa7hl8fvlb.js | 1 +
.../out/_next/static/chunks/14iajg5osx1b_.js | 1 -
.../out/_next/static/chunks/14t3iikb175xl.js | 1 +
.../out/_next/static/chunks/15b3jtv_9eob8.js | 1 +
.../out/_next/static/chunks/15nedlk5et690.css | 3 +
.../out/_next/static/chunks/181hmzmh2pbea.js | 5 -
.../out/_next/static/chunks/19dy_z32nv0iq.js | 1 +
.../out/_next/static/chunks/1_72xbmbyxrhd.js | 1 -
.../out/_next/static/chunks/1__6wkzsb58nd.js | 1 +
.../out/_next/static/chunks/1_hijls2yk428.js | 1 -
.../{3ioh_2i1gl021.js => 1_pl4k5inb4xv.js} | 2 +-
.../out/_next/static/chunks/1ajmxnpxbq3d_.js | 1 +
.../out/_next/static/chunks/1beieibo3toy7.js | 1 +
.../out/_next/static/chunks/1bg9xp4opqnt5.js | 1 +
.../out/_next/static/chunks/1bwuw0-vm6dkc.js | 31 ++
.../out/_next/static/chunks/1cr6ulv3qmjke.js | 1 +
.../out/_next/static/chunks/1cxbp_heltbkm.js | 1 +
.../out/_next/static/chunks/1d74dpsds5_zx.js | 1 +
.../out/_next/static/chunks/1dmuabgwpu2jq.js | 1 -
.../out/_next/static/chunks/1e_4bxdqx4u66.js | 1 -
.../out/_next/static/chunks/1edfvwc5_eck-.js | 31 --
.../out/_next/static/chunks/1ej24l_3227f7.js | 1 +
.../out/_next/static/chunks/1en7tcxl9dean.js | 1 +
.../out/_next/static/chunks/1epr0w1wnpysy.js | 1 -
.../out/_next/static/chunks/1fpvemj3w66ct.js | 1 +
.../out/_next/static/chunks/1gpv-xuoo10dp.js | 1 -
.../out/_next/static/chunks/1h6itt_vyh5jy.js | 1 +
.../out/_next/static/chunks/1h8n5ac96ve92.js | 38 ++
.../out/_next/static/chunks/1hxc2zyn3cfpm.js | 1 +
.../out/_next/static/chunks/1j_hyt_-pnipt.js | 1 +
.../out/_next/static/chunks/1jizn-tsklndi.js | 1 +
.../out/_next/static/chunks/1k_0mjidnorfh.js | 1 +
.../out/_next/static/chunks/1kabwy23ggmhn.js | 1 -
.../out/_next/static/chunks/1kbriss5fsqjm.js | 1 +
.../out/_next/static/chunks/1kpvojzxb_2ce.js | 2 -
.../out/_next/static/chunks/1lwjwroyz01ds.js | 1 +
.../out/_next/static/chunks/1m3jry71r_5_g.js | 1 -
.../out/_next/static/chunks/1mxw9csimvguo.js | 1 -
.../out/_next/static/chunks/1my8jj2_4u45c.js | 1 +
.../out/_next/static/chunks/1nqyvgby5okcw.js | 68 +++
.../out/_next/static/chunks/1oc43i8xactml.js | 1 +
.../out/_next/static/chunks/1oxlvxfixu1qd.js | 1 -
.../out/_next/static/chunks/1p7rizocjlzmm.js | 1 +
.../out/_next/static/chunks/1qk7h8bmx9cwy.js | 1 +
.../out/_next/static/chunks/1rsw11lcuvm5o.js | 1 +
.../out/_next/static/chunks/1sf68hiq0ae-c.js | 1 +
.../out/_next/static/chunks/1t7m5lyrljbza.js | 26 --
.../out/_next/static/chunks/1tzs0csms7cl5.js | 1 +
.../out/_next/static/chunks/1v51p_o3l31na.js | 1 +
.../out/_next/static/chunks/1vb4w9pn5k_9c.js | 1 -
.../out/_next/static/chunks/1vcdhrlx_53q_.js | 1 -
.../out/_next/static/chunks/1vjljxj58al0s.js | 1 -
.../out/_next/static/chunks/1vx0ue3bnkg7f.js | 1 -
.../out/_next/static/chunks/1wc_s6k4n6kyj.js | 1 -
.../out/_next/static/chunks/1wn25fdxme50_.js | 1 +
.../out/_next/static/chunks/1wulq3yn9yqsv.js | 1 +
.../out/_next/static/chunks/1x2zrb2defezx.js | 1 +
.../out/_next/static/chunks/1x_b27185ie7w.js | 1 -
.../out/_next/static/chunks/1xa9j9dgi3gmq.js | 1 +
.../out/_next/static/chunks/1xhfxnpj90y60.js | 405 ++++++++++++++++
.../out/_next/static/chunks/1xkrmcontg-7s.js | 1 -
.../out/_next/static/chunks/1xuknsk2a9jly.js | 1 -
.../out/_next/static/chunks/1y1bd8zcsnc-f.js | 1 +
.../out/_next/static/chunks/1ymnuz0yf1d3t.js | 49 ++
.../out/_next/static/chunks/1z8jay32-g9xt.js | 1 +
.../out/_next/static/chunks/1ztr6yvd5bte8.js | 1 +
.../out/_next/static/chunks/2-xu3m_uqnn15.js | 1 +
.../out/_next/static/chunks/204s1dxqrry1v.js | 49 --
.../out/_next/static/chunks/206aqn0z88a4d.js | 1 +
.../out/_next/static/chunks/20vezigt3zhed.js | 1 +
.../out/_next/static/chunks/22lms4uqygnld.js | 16 -
.../{0axawyhd7z6bu.js => 239g6qha-e5ln.js} | 73 +--
.../out/_next/static/chunks/23h5tyl-m5zlw.js | 1 +
.../out/_next/static/chunks/241m26i3vumgz.js | 23 +
.../out/_next/static/chunks/246p743n1gxn2.js | 14 +
.../out/_next/static/chunks/25rb7h68g_3za.js | 1 +
.../out/_next/static/chunks/25x5wlia-3twq.js | 1 -
.../out/_next/static/chunks/26lonzfqpktn-.js | 1 -
.../out/_next/static/chunks/26x4v6-0sgf35.js | 1 -
.../out/_next/static/chunks/27quqoym0jo1p.js | 1 -
.../out/_next/static/chunks/281fiazzn0ykz.js | 1 -
.../out/_next/static/chunks/289k7ubwqv36h.js | 1 -
.../out/_next/static/chunks/28fzwmvhc4sv1.js | 3 -
.../out/_next/static/chunks/292ioh33_bbx4.js | 1 -
.../out/_next/static/chunks/294dvnxgzckcv.js | 421 -----------------
.../out/_next/static/chunks/2_kecjz4xqx6-.js | 96 ----
.../out/_next/static/chunks/2aiq7su4mjaro.js | 1 -
.../out/_next/static/chunks/2ak23etir_a-u.js | 1 -
.../out/_next/static/chunks/2amk42jlrrwqu.js | 57 +++
.../out/_next/static/chunks/2b257g45-_kw_.js | 1 -
.../out/_next/static/chunks/2c5xo0brjij9o.js | 1 +
.../out/_next/static/chunks/2cw83dcuaptz_.js | 1 +
.../out/_next/static/chunks/2da7ygpq4ndo8.js | 1 -
.../out/_next/static/chunks/2dfd44r3wlgbs.js | 1 -
.../out/_next/static/chunks/2dl0dscmfpm_k.js | 1 +
.../out/_next/static/chunks/2dsu84-anah7m.js | 1 -
.../out/_next/static/chunks/2er8xjwvswjqp.js | 1 +
.../out/_next/static/chunks/2fcyfhl-hx3g1.js | 1 +
.../out/_next/static/chunks/2ft3a059vvq0y.js | 1 +
.../out/_next/static/chunks/2gbkayw_yh5ii.js | 1 -
.../out/_next/static/chunks/2gjodnwb8w7fw.js | 1 +
.../out/_next/static/chunks/2h_4-n4rgy99r.js | 5 -
.../out/_next/static/chunks/2hiwe4tay53e_.js | 1 +
.../out/_next/static/chunks/2i6wi06e8-4pi.js | 1 -
.../{0gv2z3ws304i3.js => 2i8w5pzxhj549.js} | 2 +-
.../out/_next/static/chunks/2jbh3bbu8quy4.js | 1 +
.../out/_next/static/chunks/2jxjfx8oaolav.js | 1 +
.../out/_next/static/chunks/2k4elswwq3t81.js | 1 -
.../out/_next/static/chunks/2kb5l8gm8d2xk.js | 1 +
.../out/_next/static/chunks/2kec4_eowe-l5.js | 436 ++++++++++++++++++
.../out/_next/static/chunks/2kuunyui4qzv-.js | 1 -
.../out/_next/static/chunks/2kx9rud4p4l3b.js | 1 +
.../out/_next/static/chunks/2kyjt6w0am9vr.js | 1 +
.../out/_next/static/chunks/2mq-0sx-hw8fj.js | 216 ---------
.../out/_next/static/chunks/2n-sxfwj8ir1k.js | 1 +
.../out/_next/static/chunks/2n4v-63z3zm3a.js | 1 +
.../out/_next/static/chunks/2nj37zeir5_2r.js | 1 -
.../out/_next/static/chunks/2ok-c2f3-dlxf.js | 1 -
.../out/_next/static/chunks/2p1uu5emx8nf4.js | 1 -
.../out/_next/static/chunks/2q8oe0lfniocu.js | 1 -
.../out/_next/static/chunks/2qcqdx8wuwu1-.js | 1 -
.../out/_next/static/chunks/2r1ubu07u4ro0.js | 1 +
.../out/_next/static/chunks/2rc6p1101cht_.js | 1 -
.../out/_next/static/chunks/2riseu9p5tv2u.js | 1 -
.../out/_next/static/chunks/2rrg12ws9wdmn.js | 1 -
.../out/_next/static/chunks/2rrul4dd7hn0r.js | 1 +
.../out/_next/static/chunks/2rzy9gopg5khe.js | 1 -
.../out/_next/static/chunks/2s1hyjqebuxgr.js | 1 +
.../out/_next/static/chunks/2tckjhqtu3wii.js | 1 -
.../out/_next/static/chunks/2uektj96b2c8r.js | 1 -
.../out/_next/static/chunks/2ufstil70mx7y.js | 1 +
.../out/_next/static/chunks/2wi6ubg_xifzg.js | 1 -
.../out/_next/static/chunks/2x-87ozl9sstj.js | 1 +
.../out/_next/static/chunks/2x67yv10f53lq.js | 1 -
.../out/_next/static/chunks/2xb84jlqku2i5.js | 1 +
.../out/_next/static/chunks/2xyxn7rj6t_tb.js | 1 +
.../out/_next/static/chunks/2ygf_o44mw0c7.js | 1 -
.../out/_next/static/chunks/2zjjg9kwx-prh.js | 1 -
.../out/_next/static/chunks/3-5ha3v2304bu.js | 1 +
.../out/_next/static/chunks/3-897mcmj4njz.js | 1 -
.../out/_next/static/chunks/3-ggdqew23z_1.js | 1 +
.../out/_next/static/chunks/3-v0s366kdlxu.js | 1 -
.../out/_next/static/chunks/3146e697tym4_.css | 1 -
.../out/_next/static/chunks/31nu3-bng2uoy.js | 1 +
.../out/_next/static/chunks/320ts7l5dj9yy.js | 1 +
.../out/_next/static/chunks/32qtkqblrhuih.js | 1 +
.../out/_next/static/chunks/32yp382ela_nc.js | 1 +
.../out/_next/static/chunks/33253pasn_y63.js | 1 +
.../out/_next/static/chunks/34cmpy7yd-v-f.js | 1 +
.../out/_next/static/chunks/34l2av_wbe8w8.js | 1 +
.../out/_next/static/chunks/34xeboychyu6p.js | 1 -
.../out/_next/static/chunks/358tk1ngnl1kd.js | 1 -
.../out/_next/static/chunks/36ctywk-pwzeq.js | 179 -------
.../out/_next/static/chunks/37ku8zflc54x3.js | 1 -
.../out/_next/static/chunks/38pnn2juwdhu0.js | 1 -
.../out/_next/static/chunks/393fa6ascyji4.js | 1 +
.../out/_next/static/chunks/39kbckshffz94.js | 96 ++++
.../out/_next/static/chunks/39pkav481ov23.js | 1 +
.../out/_next/static/chunks/3_l13e9g4dwql.js | 1 +
.../out/_next/static/chunks/3_lqzuqv-kb0c.js | 1 -
.../out/_next/static/chunks/3a2_ok31og9_j.js | 32 ++
.../out/_next/static/chunks/3a6q-74i0xaf0.js | 1 +
.../out/_next/static/chunks/3al7i__8s7hew.js | 1 +
.../out/_next/static/chunks/3bsf2y0yccyub.js | 12 +
.../out/_next/static/chunks/3byps95vaq7uw.js | 1 +
.../out/_next/static/chunks/3cb8zg_2ym6l7.js | 1 +
.../out/_next/static/chunks/3cdfe4ygw1_r-.js | 1 +
.../out/_next/static/chunks/3dms-ohsvzfv4.js | 1 -
.../out/_next/static/chunks/3dpan1dqc9p0i.js | 1 -
.../out/_next/static/chunks/3fj8wylwf-stx.js | 1 -
.../out/_next/static/chunks/3g5is680tc60x.js | 1 +
.../out/_next/static/chunks/3got84kam19t4.js | 1 +
.../{2zf625u-tcwdn.js => 3gwb465dhjxo9.js} | 2 +-
.../out/_next/static/chunks/3hkpazhxxi57k.js | 35 --
.../out/_next/static/chunks/3i2jjt28jqmvd.js | 1 -
.../out/_next/static/chunks/3icbpxp5y-p1_.js | 1 +
.../out/_next/static/chunks/3ies6gpj99c-3.js | 1 -
.../out/_next/static/chunks/3isv0esm685r_.js | 1 -
.../out/_next/static/chunks/3jyuhlymn08un.js | 1 -
.../out/_next/static/chunks/3kl2f49-c_ojx.js | 1 +
.../out/_next/static/chunks/3m4e65gski39t.js | 1 +
.../out/_next/static/chunks/3mhnhnyre69_q.js | 1 +
.../out/_next/static/chunks/3ml7dos958scm.js | 1 -
.../{2ygcfpfp164_o.js => 3n-4sn2-7lb89.js} | 83 ++--
.../out/_next/static/chunks/3n02s7vqnypgx.js | 1 +
.../out/_next/static/chunks/3nf-wb-846x5h.css | 1 +
.../out/_next/static/chunks/3o46x9-ng2-6l.js | 1 -
.../out/_next/static/chunks/3o66m9rh9kotz.js | 1 +
.../out/_next/static/chunks/3oo3mxuqk37z1.js | 1 +
.../out/_next/static/chunks/3p8aoxk193z4f.js | 1 -
.../{1yrn96ztx3yjc.js => 3pljm3p6kg4q6.js} | 18 +-
.../out/_next/static/chunks/3q0srap0rd2s2.js | 1 -
.../out/_next/static/chunks/3r8nfqup38ah0.js | 1 +
.../out/_next/static/chunks/3rkhwvlrs1x6n.js | 1 -
.../out/_next/static/chunks/3sy2n07rygl46.js | 1 +
.../out/_next/static/chunks/3tq9657hib0lm.js | 3 -
.../out/_next/static/chunks/3u529u1niwact.js | 1 -
.../out/_next/static/chunks/3u9qc99ytz-ri.js | 1 +
.../out/_next/static/chunks/3uor0l6dbz8m2.js | 1 -
.../out/_next/static/chunks/3us1a7skurxn9.js | 1 -
.../out/_next/static/chunks/3uv-32irom9sx.js | 1 +
.../out/_next/static/chunks/3vwj44iflrtkd.js | 5 +
.../out/_next/static/chunks/3vydbsopsf2li.js | 1 +
.../{1ikkshw7_p1qe.js => 3wo91r317_fe5.js} | 2 +-
.../out/_next/static/chunks/3wpp2gha9jamt.js | 1 +
.../out/_next/static/chunks/3xo65w_zz25u4.js | 1 -
.../out/_next/static/chunks/3yegoaduwt53n.js | 50 --
.../out/_next/static/chunks/3yhyaee1a4q65.js | 1 -
.../out/_next/static/chunks/3yx3h0y29e5rw.js | 1 +
.../out/_next/static/chunks/3zjugegu2ubgy.js | 1 -
.../out/_next/static/chunks/40quyruy3-8rk.js | 1 -
.../out/_next/static/chunks/41k0j0bj-1r2j.js | 38 --
.../out/_next/static/chunks/42kctw40_yfpi.js | 1 +
.../out/_next/static/chunks/42l1q3sduwm1n.js | 1 -
.../out/_next/static/chunks/42zu433i-e_5y.js | 1 -
.../out/_next/static/chunks/44klf7haf_-66.js | 1 -
.../out/_next/static/chunks/44p4cs0gfsy-h.js | 1 -
...4aw0u4dz.js => turbopack-1ygmeizib8n18.js} | 2 +-
.../static/media/airia.2at6jwzrkehyr.svg | 14 +
...fqk9ncz.svg => cerebras.10a3ay2mhz9lq.svg} | 178 +++----
.../media/claude-code.0mamz0si_fpue.svg | 1 +
.../static/media/codex.3urhv_jmx5kj5.svg | 1 +
.../media/crewai-color.067lq5ci-ka84.svg | 1 +
.../static/media/edenai.3z_rirzycrnpm.svg | 5 +
.../static/media/favicon.34m4ev7_7ennf.ico | Bin 0 -> 4495 bytes
.../static/media/favicon.3arlap5n8tyzg.ico | Bin 6387 -> 0 bytes
.../static/media/google-adk.00m40sq68bb9e.png | Bin 0 -> 11976 bytes
.../static/media/hermes.1b_2kl2itiouq.png | Bin 0 -> 1100 bytes
.../static/media/langchain.1p5ul5un_z7te.svg | 1 +
.../media/langgraph-color.2y5v3012kup9-.svg | 1 +
.../media/lens-symbol.1k7qocul2grzx.svg | 3 +
.../media/litellm_logo.2q-1n9v95d189.jpg | Bin 9222 -> 0 bytes
.../media/litellm_monogram.16r2_rdj4dq8r.svg | 17 +
.../media/llamaindex-color.0v6x7n2qxbs8x.svg | 1 +
.../media/llm_shield_proxy.1mr3gmqtwon8d.svg | 6 +
.../microsoft_365_copilot.1jztrfb78qq1x.svg | 43 ++
.../static/media/moyai-head.3-tcfui7g9zwr.svg | 19 +
.../media/openai-agents.0t1sdv57nux6g.svg | 1 +
.../static/media/openai.3rj8nedgjpevh.svg | 5 +
.../static/media/openclaw.05tk6t9ff_l47.png | Bin 0 -> 1015 bytes
.../static/media/opencode.1hqe_7u8rr8ab.svg | 16 +
.../media/opentelemetry.2n8ujs5z4l__r.svg | 1 +
.../media/pydantic-ai-color.1f334m-pf8a75.svg | 1 +
.../static/media/signoz.2x1bmq4-si8uu.svg | 1 +
.../static/media/strands.1w9vvik2r671z.svg | 4 +
.../static/media/tencent.1mqjwxxk-obwq.svg | 6 +
.../static/media/typesafe.44kuy2p41yhxx.png | Bin 0 -> 4311 bytes
.../static/media/wingback.1em0mjtwauacr.png | Bin 0 -> 14286 bytes
.../_buildManifest.js | 0
.../_clientMiddlewareManifest.js | 0
.../_ssgManifest.js | 0
.../out/_not-found/__next._full.txt | 43 +-
.../_not-found/__next._not-found.__PAGE__.txt | 35 +-
.../out/_not-found/__next._tree.txt | 6 +-
.../_experimental/out/_not-found/index.html | 2 +-
.../_experimental/out/_not-found/index.txt | 43 +-
...KGRhc2hib2FyZCk.access-groups.__PAGE__.txt | 55 ++-
.../out/access-groups/__next._full.txt | 70 +--
.../out/access-groups/__next._tree.txt | 6 +-
.../out/access-groups/index.html | 2 +-
.../_experimental/out/access-groups/index.txt | 70 +--
....!KGRhc2hib2FyZCk.admin-panel.__PAGE__.txt | 55 ++-
.../out/admin-panel/__next._full.txt | 70 +--
.../out/admin-panel/__next._tree.txt | 6 +-
.../_experimental/out/admin-panel/index.html | 2 +-
.../_experimental/out/admin-panel/index.txt | 70 +--
..._next.!KGRhc2hib2FyZCk.agents.__PAGE__.txt | 57 ++-
.../_experimental/out/agents/__next._full.txt | 70 +--
.../_experimental/out/agents/__next._tree.txt | 6 +-
.../proxy/_experimental/out/agents/index.html | 2 +-
.../proxy/_experimental/out/agents/index.txt | 70 +--
...ext.!KGRhc2hib2FyZCk.api-keys.__PAGE__.txt | 48 +-
.../out/api-keys/__next._full.txt | 70 +--
.../out/api-keys/__next._tree.txt | 6 +-
.../_experimental/out/api-keys/index.html | 2 +-
.../_experimental/out/api-keys/index.txt | 70 +--
...KGRhc2hib2FyZCk.api-reference.__PAGE__.txt | 39 +-
.../out/api-reference/__next._full.txt | 70 +--
.../out/api-reference/__next._tree.txt | 6 +-
.../out/api-reference/index.html | 2 +-
.../_experimental/out/api-reference/index.txt | 70 +--
.../_experimental/out/assets/logos/airia.svg | 14 +
.../out/assets/logos/cerebras.svg | 178 +++----
.../out/assets/logos/crewai-color.svg | 1 +
.../_experimental/out/assets/logos/edenai.svg | 5 +
.../out/assets/logos/google-adk.png | Bin 0 -> 11976 bytes
.../_experimental/out/assets/logos/hermes.png | Bin 0 -> 1100 bytes
.../out/assets/logos/langchain.svg | 1 +
.../out/assets/logos/langgraph-color.svg | 1 +
.../out/assets/logos/litellm_logo.jpg | Bin 9222 -> 0 bytes
.../out/assets/logos/llamaindex-color.svg | 1 +
.../out/assets/logos/llm_shield_proxy.svg | 6 +
.../assets/logos/microsoft_365_copilot.svg | 43 ++
.../out/assets/logos/openai-agents.svg | 1 +
.../out/assets/logos/openclaw.png | Bin 0 -> 1015 bytes
.../out/assets/logos/opentelemetry.svg | 1 +
.../out/assets/logos/pydantic-ai-color.svg | 1 +
.../out/assets/logos/strands.svg | 4 +
.../out/assets/logos/tencent.svg | 6 +
.../out/assets/logos/typesafe.png | Bin 0 -> 4311 bytes
.../out/assets/logos/wingback.png | Bin 0 -> 14286 bytes
.../out/assets/moyai/logos/anthropic.svg | 5 +
.../out/assets/moyai/logos/bedrock.svg | 1 +
.../out/assets/moyai/logos/deepseek.svg | 25 +
.../out/assets/moyai/logos/fireworks.svg | 1 +
.../out/assets/moyai/logos/google.svg | 2 +
.../out/assets/moyai/logos/hermes.png | Bin 0 -> 1100 bytes
.../out/assets/moyai/logos/langchain.svg | 1 +
.../out/assets/moyai/logos/mistral.svg | 1 +
.../out/assets/moyai/logos/openai.svg | 5 +
.../out/assets/moyai/logos/opencode.svg | 16 +
.../out/assets/moyai/logos/xai.svg | 28 ++
.../out/assets/moyai/moyai-head.svg | 19 +
...next.!KGRhc2hib2FyZCk.budgets.__PAGE__.txt | 55 ++-
.../out/budgets/__next._full.txt | 70 +--
.../out/budgets/__next._tree.txt | 6 +-
.../_experimental/out/budgets/index.html | 2 +-
.../proxy/_experimental/out/budgets/index.txt | 70 +--
...next.!KGRhc2hib2FyZCk.caching.__PAGE__.txt | 52 ++-
.../out/caching/__next._full.txt | 70 +--
.../out/caching/__next._tree.txt | 6 +-
.../_experimental/out/caching/index.html | 2 +-
.../proxy/_experimental/out/caching/index.txt | 70 +--
...Rhc2hib2FyZCk.change-password.__PAGE__.txt | 38 ++
.../out/change-password/__next._full.txt | 39 ++
.../out/change-password/__next._tree.txt | 4 +
.../out/change-password/index.html | 1 +
.../out/change-password/index.txt | 39 ++
.../_experimental/out/chat/__next._full.txt | 62 +--
.../_experimental/out/chat/__next._tree.txt | 6 +-
.../out/chat/__next.chat.__PAGE__.txt | 48 +-
.../out/chat/api-keys/__next._full.txt | 60 +--
.../out/chat/api-keys/__next._tree.txt | 6 +-
.../__next.chat.api-keys.__PAGE__.txt | 44 +-
.../out/chat/api-keys/index.html | 2 +-
.../_experimental/out/chat/api-keys/index.txt | 60 +--
.../out/chat/credentials/__next._full.txt | 60 +--
.../out/chat/credentials/__next._tree.txt | 6 +-
.../__next.chat.credentials.__PAGE__.txt | 44 +-
.../out/chat/credentials/index.html | 2 +-
.../out/chat/credentials/index.txt | 60 +--
.../proxy/_experimental/out/chat/index.html | 2 +-
.../proxy/_experimental/out/chat/index.txt | 62 +--
.../out/chat/integrations/__next._full.txt | 61 +--
.../out/chat/integrations/__next._tree.txt | 6 +-
.../__next.chat.integrations.__PAGE__.txt | 45 +-
.../out/chat/integrations/index.html | 2 +-
.../out/chat/integrations/index.txt | 61 +--
.../out/chat/logs/__next._full.txt | 64 +--
.../out/chat/logs/__next._tree.txt | 6 +-
.../chat/logs/__next.chat.logs.__PAGE__.txt | 45 +-
.../_experimental/out/chat/logs/index.html | 2 +-
.../_experimental/out/chat/logs/index.txt | 64 +--
.../out/chat/usage/__next._full.txt | 59 +--
.../out/chat/usage/__next._tree.txt | 6 +-
.../chat/usage/__next.chat.usage.__PAGE__.txt | 44 +-
.../_experimental/out/chat/usage/index.html | 2 +-
.../_experimental/out/chat/usage/index.txt | 59 +--
.../out/connect/__next._full.txt | 53 +--
.../out/connect/__next._tree.txt | 6 +-
.../out/connect/__next.connect.__PAGE__.txt | 43 +-
.../_experimental/out/connect/index.html | 2 +-
.../proxy/_experimental/out/connect/index.txt | 53 +--
...c2hib2FyZCk.cost-optimization.__PAGE__.txt | 51 +-
.../out/cost-optimization/__next._full.txt | 70 +--
.../out/cost-optimization/__next._tree.txt | 6 +-
.../out/cost-optimization/index.html | 2 +-
.../out/cost-optimization/index.txt | 70 +--
...KGRhc2hib2FyZCk.cost-tracking.__PAGE__.txt | 45 +-
.../out/cost-tracking/__next._full.txt | 70 +--
.../out/cost-tracking/__next._tree.txt | 6 +-
.../out/cost-tracking/index.html | 2 +-
.../_experimental/out/cost-tracking/index.txt | 70 +--
...2hib2FyZCk.guardrails-monitor.__PAGE__.txt | 53 ++-
.../out/guardrails-monitor/__next._full.txt | 70 +--
.../out/guardrails-monitor/__next._tree.txt | 6 +-
.../out/guardrails-monitor/index.html | 2 +-
.../out/guardrails-monitor/index.txt | 70 +--
...t.!KGRhc2hib2FyZCk.guardrails.__PAGE__.txt | 50 +-
.../out/guardrails/__next._full.txt | 70 +--
.../out/guardrails/__next._tree.txt | 6 +-
.../_experimental/out/guardrails/index.html | 2 +-
.../_experimental/out/guardrails/index.txt | 70 +--
.../__next.!KGRhc2hib2FyZCk.home.__PAGE__.txt | 40 ++
.../_experimental/out/home/__next._full.txt | 39 ++
.../_experimental/out/home/__next._tree.txt | 4 +
.../proxy/_experimental/out/home/index.html | 1 +
.../proxy/_experimental/out/home/index.txt | 39 ++
litellm/proxy/_experimental/out/index.html | 2 +-
litellm/proxy/_experimental/out/index.txt | 64 +--
.../__next.!KGRhc2hib2FyZCk.lens.__PAGE__.txt | 41 ++
.../_experimental/out/lens/__next._full.txt | 41 ++
.../_experimental/out/lens/__next._tree.txt | 6 +
.../proxy/_experimental/out/lens/index.html | 1 +
.../proxy/_experimental/out/lens/index.txt | 41 ++
...2hib2FyZCk.logging-and-alerts.__PAGE__.txt | 55 ++-
.../out/logging-and-alerts/__next._full.txt | 70 +--
.../out/logging-and-alerts/__next._tree.txt | 6 +-
.../out/logging-and-alerts/index.html | 2 +-
.../out/logging-and-alerts/index.txt | 70 +--
.../_experimental/out/login/__next._full.txt | 51 +-
.../_experimental/out/login/__next._tree.txt | 6 +-
.../out/login/__next.login.__PAGE__.txt | 39 +-
.../proxy/_experimental/out/login/index.html | 2 +-
.../proxy/_experimental/out/login/index.txt | 51 +-
.../__next.!KGRhc2hib2FyZCk.logs.__PAGE__.txt | 59 +--
.../_experimental/out/logs/__next._full.txt | 70 +--
.../_experimental/out/logs/__next._tree.txt | 6 +-
.../proxy/_experimental/out/logs/index.html | 2 +-
.../proxy/_experimental/out/logs/index.txt | 70 +--
....!KGRhc2hib2FyZCk.mcp-servers.__PAGE__.txt | 50 +-
.../out/mcp-servers/__next._full.txt | 70 +--
.../out/mcp-servers/__next._tree.txt | 6 +-
.../_experimental/out/mcp-servers/index.html | 2 +-
.../_experimental/out/mcp-servers/index.txt | 70 +--
.../out/mcp/oauth/callback/__next._full.txt | 51 +-
.../out/mcp/oauth/callback/__next._tree.txt | 6 +-
.../__next.mcp.oauth.callback.__PAGE__.txt | 39 +-
.../out/mcp/oauth/callback/index.html | 2 +-
.../out/mcp/oauth/callback/index.txt | 51 +-
..._next.!KGRhc2hib2FyZCk.memory.__PAGE__.txt | 47 +-
.../_experimental/out/memory/__next._full.txt | 70 +--
.../_experimental/out/memory/__next._tree.txt | 6 +-
.../proxy/_experimental/out/memory/index.html | 2 +-
.../proxy/_experimental/out/memory/index.txt | 70 +--
...Rhc2hib2FyZCk.model-hub-table.__PAGE__.txt | 50 +-
.../out/model-hub-table/__next._full.txt | 70 +--
.../out/model-hub-table/__next._tree.txt | 6 +-
.../out/model-hub-table/index.html | 2 +-
.../out/model-hub-table/index.txt | 70 +--
...GRhc2hib2FyZCk.model-insights.__PAGE__.txt | 39 ++
.../out/model-insights/__next._full.txt | 39 ++
.../out/model-insights/__next._tree.txt | 4 +
.../out/model-insights/index.html | 1 +
.../out/model-insights/index.txt | 39 ++
.../out/model_hub/__next._full.txt | 58 +--
.../out/model_hub/__next._tree.txt | 6 +-
.../model_hub/__next.model_hub.__PAGE__.txt | 44 +-
.../_experimental/out/model_hub/index.html | 2 +-
.../_experimental/out/model_hub/index.txt | 58 +--
.../out/model_hub_table/__next._full.txt | 62 +--
.../out/model_hub_table/__next._tree.txt | 6 +-
.../__next.model_hub_table.__PAGE__.txt | 45 +-
.../out/model_hub_table/index.html | 2 +-
.../out/model_hub_table/index.txt | 62 +--
...ib2FyZCk.models-and-endpoints.__PAGE__.txt | 61 +--
.../out/models-and-endpoints/__next._full.txt | 71 +--
.../out/models-and-endpoints/__next._tree.txt | 6 +-
.../out/models-and-endpoints/index.html | 2 +-
.../out/models-and-endpoints/index.txt | 71 +--
.../_experimental/out/moyai/__next._full.txt | 29 ++
.../_experimental/out/moyai/__next._tree.txt | 5 +
.../out/moyai/__next.moyai.__PAGE__.txt | 34 ++
.../proxy/_experimental/out/moyai/index.html | 2 +
.../proxy/_experimental/out/moyai/index.txt | 29 ++
...xt.!KGRhc2hib2FyZCk.old-usage.__PAGE__.txt | 48 +-
.../out/old-usage/__next._full.txt | 70 +--
.../out/old-usage/__next._tree.txt | 6 +-
.../_experimental/out/old-usage/index.html | 2 +-
.../_experimental/out/old-usage/index.txt | 70 +--
.../out/onboarding/__next._full.txt | 51 +-
.../out/onboarding/__next._tree.txt | 6 +-
.../onboarding/__next.onboarding.__PAGE__.txt | 39 +-
.../_experimental/out/onboarding/index.html | 2 +-
.../_experimental/out/onboarding/index.txt | 51 +-
...KGRhc2hib2FyZCk.organizations.__PAGE__.txt | 50 +-
.../out/organizations/__next._full.txt | 70 +--
.../out/organizations/__next._tree.txt | 6 +-
.../out/organizations/index.html | 2 +-
.../_experimental/out/organizations/index.txt | 70 +--
...t.!KGRhc2hib2FyZCk.playground.__PAGE__.txt | 51 +-
.../out/playground/__next._full.txt | 70 +--
.../out/playground/__next._tree.txt | 6 +-
.../_experimental/out/playground/index.html | 2 +-
.../_experimental/out/playground/index.txt | 70 +--
...ext.!KGRhc2hib2FyZCk.policies.__PAGE__.txt | 51 +-
.../out/policies/__next._full.txt | 70 +--
.../out/policies/__next._tree.txt | 6 +-
.../_experimental/out/policies/index.html | 2 +-
.../_experimental/out/policies/index.txt | 70 +--
...ext.!KGRhc2hib2FyZCk.projects.__PAGE__.txt | 51 +-
.../out/projects/__next._full.txt | 70 +--
.../out/projects/__next._tree.txt | 6 +-
.../_experimental/out/projects/index.html | 2 +-
.../_experimental/out/projects/index.txt | 70 +--
...next.!KGRhc2hib2FyZCk.prompts.__PAGE__.txt | 51 +-
.../out/prompts/__next._full.txt | 70 +--
.../out/prompts/__next._tree.txt | 6 +-
.../_experimental/out/prompts/index.html | 2 +-
.../proxy/_experimental/out/prompts/index.txt | 70 +--
...Rhc2hib2FyZCk.router-settings.__PAGE__.txt | 52 ++-
.../out/router-settings/__next._full.txt | 70 +--
.../out/router-settings/__next._tree.txt | 6 +-
.../out/router-settings/index.html | 2 +-
.../out/router-settings/index.txt | 70 +--
...!KGRhc2hib2FyZCk.search-tools.__PAGE__.txt | 54 +--
.../out/search-tools/__next._full.txt | 70 +--
.../out/search-tools/__next._tree.txt | 6 +-
.../_experimental/out/search-tools/index.html | 2 +-
.../_experimental/out/search-tools/index.txt | 70 +--
..._next.!KGRhc2hib2FyZCk.skills.__PAGE__.txt | 47 +-
.../_experimental/out/skills/__next._full.txt | 70 +--
.../_experimental/out/skills/__next._tree.txt | 6 +-
.../proxy/_experimental/out/skills/index.html | 2 +-
.../proxy/_experimental/out/skills/index.txt | 70 +--
...GRhc2hib2FyZCk.tag-management.__PAGE__.txt | 50 +-
.../out/tag-management/__next._full.txt | 70 +--
.../out/tag-management/__next._tree.txt | 6 +-
.../out/tag-management/index.html | 2 +-
.../out/tag-management/index.txt | 70 +--
...__next.!KGRhc2hib2FyZCk.teams.__PAGE__.txt | 61 +--
.../_experimental/out/teams/__next._full.txt | 70 +--
.../_experimental/out/teams/__next._tree.txt | 6 +-
.../proxy/_experimental/out/teams/index.html | 2 +-
.../proxy/_experimental/out/teams/index.txt | 70 +--
...xt.!KGRhc2hib2FyZCk.telemetry.__PAGE__.txt | 38 ++
.../out/telemetry/__next._full.txt | 39 ++
.../out/telemetry/__next._tree.txt | 4 +
.../_experimental/out/telemetry/index.html | 1 +
.../_experimental/out/telemetry/index.txt | 39 ++
...KGRhc2hib2FyZCk.tool-policies.__PAGE__.txt | 50 +-
.../out/tool-policies/__next._full.txt | 70 +--
.../out/tool-policies/__next._tree.txt | 6 +-
.../out/tool-policies/index.html | 2 +-
.../_experimental/out/tool-policies/index.txt | 70 +--
...c2hib2FyZCk.transform-request.__PAGE__.txt | 45 +-
.../out/transform-request/__next._full.txt | 70 +--
.../out/transform-request/__next._tree.txt | 6 +-
.../out/transform-request/index.html | 2 +-
.../out/transform-request/index.txt | 70 +--
...ext.!KGRhc2hib2FyZCk.ui-theme.__PAGE__.txt | 45 +-
.../out/ui-theme/__next._full.txt | 70 +--
.../out/ui-theme/__next._tree.txt | 6 +-
.../_experimental/out/ui-theme/index.html | 2 +-
.../_experimental/out/ui-theme/index.txt | 70 +--
...__next.!KGRhc2hib2FyZCk.usage.__PAGE__.txt | 55 ++-
.../_experimental/out/usage/__next._full.txt | 70 +--
.../_experimental/out/usage/__next._tree.txt | 6 +-
.../proxy/_experimental/out/usage/index.html | 2 +-
.../proxy/_experimental/out/usage/index.txt | 70 +--
...__next.!KGRhc2hib2FyZCk.users.__PAGE__.txt | 50 +-
.../_experimental/out/users/__next._full.txt | 70 +--
.../_experimental/out/users/__next._tree.txt | 6 +-
.../proxy/_experimental/out/users/index.html | 2 +-
.../proxy/_experimental/out/users/index.txt | 70 +--
...KGRhc2hib2FyZCk.vector-stores.__PAGE__.txt | 55 ++-
.../out/vector-stores/__next._full.txt | 70 +--
.../out/vector-stores/__next._tree.txt | 6 +-
.../out/vector-stores/index.html | 2 +-
.../_experimental/out/vector-stores/index.txt | 70 +--
...xt.!KGRhc2hib2FyZCk.workflows.__PAGE__.txt | 50 +-
.../out/workflows/__next._full.txt | 70 +--
.../out/workflows/__next._tree.txt | 6 +-
.../_experimental/out/workflows/index.html | 2 +-
.../_experimental/out/workflows/index.txt | 70 +--
.../check_no_publicly_known_master_key.py | 3 +-
...test_check_no_publicly_known_master_key.py | 22 +-
668 files changed, 8682 insertions(+), 6609 deletions(-)
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0-p31wd20yje5.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0-w0rr3df0htc.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0-xrvklzr286n.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/006y36jxl8z-u.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/01cjt3b43zsyy.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/01f03qwhd6l7d.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/01msi0ehnlc7g.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/02erlqrgtvdvz.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/02mlplp0iptro.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/033urjy22ackz.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/034i32t0-7tdv.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/03kp2as6u_s_6.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/03xf0a_nt1mqx.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/04m1lhogzlu_q.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/058x5ogyudznz.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05cei057fcv33.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05evi9aern8_5.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05p3egzpg0fru.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/05php4kcqbp33.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0677cowehlx_9.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/06el5-9ee247i.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/06yuul4yvgbd0.css
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/07ifwp0wdqloz.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/07xbdb1bjx9eg.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/08x62apceyttf.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/08x8fsl2hzbmq.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/095t3bxug2-v_.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/09kfqcp7rqvl7.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/09pcs5yy22ada.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/09qlj1_ya5uqw.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/09tfr__sbh796.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0_50gw2o6ba0j.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0_8vcd9i7eo1r.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0__ufucx2g6ui.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0_t1_1-2to_0w.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ao344k1l0l2h.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0bhhnflodjdhp.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0bks94633rs4s.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0bl98f8jm71jn.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0blpz77qmgyho.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ca8wd799xeut.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{1kid9zr1--h6y.css => 0ckv5j_fni3xm.css} (96%)
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0d-myyz3fnj3y.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0dyoztesepp-8.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ec5zmg5_3qwx.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0eclgobdwjfbw.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0eh6yvm8qswse.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0el1dng5997zf.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0eruvlogn4q8l.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ev0y17rt0bzh.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0eybcbrej9bl8.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0f4daktg2mnzj.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0f74g4_7j-4ju.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0gjxeo9f--vzt.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0h6b6ooi-yfmn.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0hukc5zl2txhg.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0i--ursme41qh.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0iad0mpbdl5k9.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0icwvr3xvxvlx.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0j0877r_j866g.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{2atns3zer6s42.js => 0jau1crjcgz3m.js} (92%)
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ldd7ocximwhh.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0md57zg_zhxqq.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0n3xjld_n2chp.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0nqgb35_9l922.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0o2yjrr5g_tdw.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0och7iysqbm0w.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0pz3k7bzm5al8.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0q6x33fc-tn6m.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0qlyu_3ohy0_9.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0r0nhtsbxio43.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0r2no56zz5i7e.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0refy__7v6-qb.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0rq646fx4-bql.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0sn6ne06gs8iu.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0stffhbqahki3.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{0_u7rrnrqu95t.js => 0svv6cnxs4pbi.js} (68%)
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0sz89fsnzc09a.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0t3a_qboss-93.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0tzl5rama7x4_.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ucu5tumma10r.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ui61y5hgz0ck.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0uo3fyy_3adzw.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0vpn3th7sn4vf.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0w5e2qyxu1sj4.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0wk0l0tbb11zh.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0x88jgebq4fjq.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0xfewjecn0ixz.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0xl3regan_n7s.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0xsuy-8_q50ub.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0ye1nryroojck.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0zi1szq8x_gmi.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/0zm9m2grnntd3.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1-metsezi443m.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1-zxupj2b23jk.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/10i7gzq_kbeak.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/115x4nuphlkvv.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/11c-g4jel910l.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/11eb6uxtl-k7c.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/11o_e34ji0wx-.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/11vytukfj5_7x.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/12_-wfvirgolu.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/12thj6aa-tc5c.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{0lwia0t_dwgb-.js => 13douredghzbr.js} (72%)
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/13lpebw2fp-wr.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/140374h5u57ha.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/142sa7hl8fvlb.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/14iajg5osx1b_.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/14t3iikb175xl.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/15b3jtv_9eob8.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/15nedlk5et690.css
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/181hmzmh2pbea.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/19dy_z32nv0iq.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1_72xbmbyxrhd.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1__6wkzsb58nd.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1_hijls2yk428.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{3ioh_2i1gl021.js => 1_pl4k5inb4xv.js} (94%)
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1ajmxnpxbq3d_.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1beieibo3toy7.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1bg9xp4opqnt5.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1bwuw0-vm6dkc.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1cr6ulv3qmjke.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1cxbp_heltbkm.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1d74dpsds5_zx.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1dmuabgwpu2jq.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1e_4bxdqx4u66.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1edfvwc5_eck-.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1ej24l_3227f7.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1en7tcxl9dean.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1epr0w1wnpysy.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1fpvemj3w66ct.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1gpv-xuoo10dp.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1h6itt_vyh5jy.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1h8n5ac96ve92.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1hxc2zyn3cfpm.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1j_hyt_-pnipt.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1jizn-tsklndi.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1k_0mjidnorfh.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1kabwy23ggmhn.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1kbriss5fsqjm.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1kpvojzxb_2ce.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1lwjwroyz01ds.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1m3jry71r_5_g.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1mxw9csimvguo.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1my8jj2_4u45c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1nqyvgby5okcw.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1oc43i8xactml.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1oxlvxfixu1qd.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1p7rizocjlzmm.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1qk7h8bmx9cwy.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1rsw11lcuvm5o.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1sf68hiq0ae-c.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1t7m5lyrljbza.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1tzs0csms7cl5.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1v51p_o3l31na.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1vb4w9pn5k_9c.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1vcdhrlx_53q_.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1vjljxj58al0s.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1vx0ue3bnkg7f.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1wc_s6k4n6kyj.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1wn25fdxme50_.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1wulq3yn9yqsv.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1x2zrb2defezx.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1x_b27185ie7w.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1xa9j9dgi3gmq.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1xhfxnpj90y60.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1xkrmcontg-7s.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1xuknsk2a9jly.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1y1bd8zcsnc-f.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1ymnuz0yf1d3t.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1z8jay32-g9xt.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1ztr6yvd5bte8.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2-xu3m_uqnn15.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/204s1dxqrry1v.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/206aqn0z88a4d.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/20vezigt3zhed.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/22lms4uqygnld.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{0axawyhd7z6bu.js => 239g6qha-e5ln.js} (60%)
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/23h5tyl-m5zlw.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/241m26i3vumgz.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/246p743n1gxn2.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/25rb7h68g_3za.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/25x5wlia-3twq.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/26lonzfqpktn-.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/26x4v6-0sgf35.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/27quqoym0jo1p.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/281fiazzn0ykz.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/289k7ubwqv36h.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/28fzwmvhc4sv1.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/292ioh33_bbx4.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/294dvnxgzckcv.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2_kecjz4xqx6-.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2aiq7su4mjaro.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2ak23etir_a-u.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2amk42jlrrwqu.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2b257g45-_kw_.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2c5xo0brjij9o.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2cw83dcuaptz_.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2da7ygpq4ndo8.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2dfd44r3wlgbs.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2dl0dscmfpm_k.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2dsu84-anah7m.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2er8xjwvswjqp.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2fcyfhl-hx3g1.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2ft3a059vvq0y.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2gbkayw_yh5ii.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2gjodnwb8w7fw.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2h_4-n4rgy99r.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2hiwe4tay53e_.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2i6wi06e8-4pi.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{0gv2z3ws304i3.js => 2i8w5pzxhj549.js} (75%)
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2jbh3bbu8quy4.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2jxjfx8oaolav.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2k4elswwq3t81.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2kb5l8gm8d2xk.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2kec4_eowe-l5.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2kuunyui4qzv-.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2kx9rud4p4l3b.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2kyjt6w0am9vr.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2mq-0sx-hw8fj.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2n-sxfwj8ir1k.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2n4v-63z3zm3a.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2nj37zeir5_2r.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2ok-c2f3-dlxf.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2p1uu5emx8nf4.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2q8oe0lfniocu.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2qcqdx8wuwu1-.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2r1ubu07u4ro0.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2rc6p1101cht_.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2riseu9p5tv2u.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2rrg12ws9wdmn.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2rrul4dd7hn0r.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2rzy9gopg5khe.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2s1hyjqebuxgr.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2tckjhqtu3wii.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2uektj96b2c8r.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2ufstil70mx7y.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2wi6ubg_xifzg.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2x-87ozl9sstj.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2x67yv10f53lq.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2xb84jlqku2i5.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2xyxn7rj6t_tb.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2ygf_o44mw0c7.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2zjjg9kwx-prh.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3-5ha3v2304bu.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3-897mcmj4njz.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3-ggdqew23z_1.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3-v0s366kdlxu.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3146e697tym4_.css
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/31nu3-bng2uoy.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/320ts7l5dj9yy.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/32qtkqblrhuih.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/32yp382ela_nc.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/33253pasn_y63.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/34cmpy7yd-v-f.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/34l2av_wbe8w8.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/34xeboychyu6p.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/358tk1ngnl1kd.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/36ctywk-pwzeq.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/37ku8zflc54x3.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/38pnn2juwdhu0.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/393fa6ascyji4.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/39kbckshffz94.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/39pkav481ov23.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3_l13e9g4dwql.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3_lqzuqv-kb0c.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3a2_ok31og9_j.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3a6q-74i0xaf0.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3al7i__8s7hew.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3bsf2y0yccyub.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3byps95vaq7uw.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3cb8zg_2ym6l7.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3cdfe4ygw1_r-.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3dms-ohsvzfv4.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3dpan1dqc9p0i.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3fj8wylwf-stx.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3g5is680tc60x.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3got84kam19t4.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{2zf625u-tcwdn.js => 3gwb465dhjxo9.js} (99%)
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3hkpazhxxi57k.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3i2jjt28jqmvd.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3icbpxp5y-p1_.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3ies6gpj99c-3.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3isv0esm685r_.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3jyuhlymn08un.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3kl2f49-c_ojx.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3m4e65gski39t.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3mhnhnyre69_q.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3ml7dos958scm.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{2ygcfpfp164_o.js => 3n-4sn2-7lb89.js} (59%)
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3n02s7vqnypgx.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3nf-wb-846x5h.css
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3o46x9-ng2-6l.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3o66m9rh9kotz.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3oo3mxuqk37z1.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3p8aoxk193z4f.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{1yrn96ztx3yjc.js => 3pljm3p6kg4q6.js} (62%)
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3q0srap0rd2s2.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3r8nfqup38ah0.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3rkhwvlrs1x6n.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3sy2n07rygl46.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3tq9657hib0lm.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3u529u1niwact.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3u9qc99ytz-ri.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3uor0l6dbz8m2.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3us1a7skurxn9.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3uv-32irom9sx.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3vwj44iflrtkd.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3vydbsopsf2li.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{1ikkshw7_p1qe.js => 3wo91r317_fe5.js} (69%)
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3wpp2gha9jamt.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3xo65w_zz25u4.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3yegoaduwt53n.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3yhyaee1a4q65.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3yx3h0y29e5rw.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3zjugegu2ubgy.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/40quyruy3-8rk.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/41k0j0bj-1r2j.js
create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/42kctw40_yfpi.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/42l1q3sduwm1n.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/42zu433i-e_5y.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/44klf7haf_-66.js
delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/44p4cs0gfsy-h.js
rename litellm/proxy/_experimental/out/_next/static/chunks/{turbopack-2-a_44aw0u4dz.js => turbopack-1ygmeizib8n18.js} (90%)
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/airia.2at6jwzrkehyr.svg
rename litellm/proxy/_experimental/out/_next/static/media/{cerebras.1ur1xyfqk9ncz.svg => cerebras.10a3ay2mhz9lq.svg} (97%)
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/claude-code.0mamz0si_fpue.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/codex.3urhv_jmx5kj5.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/crewai-color.067lq5ci-ka84.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/edenai.3z_rirzycrnpm.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/favicon.34m4ev7_7ennf.ico
delete mode 100644 litellm/proxy/_experimental/out/_next/static/media/favicon.3arlap5n8tyzg.ico
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/google-adk.00m40sq68bb9e.png
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/hermes.1b_2kl2itiouq.png
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/langchain.1p5ul5un_z7te.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/langgraph-color.2y5v3012kup9-.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/lens-symbol.1k7qocul2grzx.svg
delete mode 100644 litellm/proxy/_experimental/out/_next/static/media/litellm_logo.2q-1n9v95d189.jpg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/litellm_monogram.16r2_rdj4dq8r.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/llamaindex-color.0v6x7n2qxbs8x.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/llm_shield_proxy.1mr3gmqtwon8d.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/microsoft_365_copilot.1jztrfb78qq1x.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/moyai-head.3-tcfui7g9zwr.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/openai-agents.0t1sdv57nux6g.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/openai.3rj8nedgjpevh.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/openclaw.05tk6t9ff_l47.png
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/opencode.1hqe_7u8rr8ab.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/opentelemetry.2n8ujs5z4l__r.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/pydantic-ai-color.1f334m-pf8a75.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/signoz.2x1bmq4-si8uu.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/strands.1w9vvik2r671z.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/tencent.1mqjwxxk-obwq.svg
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/typesafe.44kuy2p41yhxx.png
create mode 100644 litellm/proxy/_experimental/out/_next/static/media/wingback.1em0mjtwauacr.png
rename litellm/proxy/_experimental/out/_next/static/{kXnLzJ6ylsRPmgSkCkCKM => u_MWVLx_WA9i-AM2JdCsp}/_buildManifest.js (100%)
rename litellm/proxy/_experimental/out/_next/static/{kXnLzJ6ylsRPmgSkCkCKM => u_MWVLx_WA9i-AM2JdCsp}/_clientMiddlewareManifest.js (100%)
rename litellm/proxy/_experimental/out/_next/static/{kXnLzJ6ylsRPmgSkCkCKM => u_MWVLx_WA9i-AM2JdCsp}/_ssgManifest.js (100%)
create mode 100644 litellm/proxy/_experimental/out/assets/logos/airia.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/crewai-color.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/edenai.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/google-adk.png
create mode 100644 litellm/proxy/_experimental/out/assets/logos/hermes.png
create mode 100644 litellm/proxy/_experimental/out/assets/logos/langchain.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/langgraph-color.svg
delete mode 100644 litellm/proxy/_experimental/out/assets/logos/litellm_logo.jpg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/llamaindex-color.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/llm_shield_proxy.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/microsoft_365_copilot.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/openai-agents.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/openclaw.png
create mode 100644 litellm/proxy/_experimental/out/assets/logos/opentelemetry.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/pydantic-ai-color.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/strands.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/tencent.svg
create mode 100644 litellm/proxy/_experimental/out/assets/logos/typesafe.png
create mode 100644 litellm/proxy/_experimental/out/assets/logos/wingback.png
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/anthropic.svg
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/bedrock.svg
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/deepseek.svg
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/fireworks.svg
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/google.svg
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/hermes.png
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/langchain.svg
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/mistral.svg
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/openai.svg
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/opencode.svg
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/logos/xai.svg
create mode 100644 litellm/proxy/_experimental/out/assets/moyai/moyai-head.svg
create mode 100644 litellm/proxy/_experimental/out/change-password/__next.!KGRhc2hib2FyZCk.change-password.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/change-password/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/change-password/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/change-password/index.html
create mode 100644 litellm/proxy/_experimental/out/change-password/index.txt
create mode 100644 litellm/proxy/_experimental/out/home/__next.!KGRhc2hib2FyZCk.home.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/home/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/home/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/home/index.html
create mode 100644 litellm/proxy/_experimental/out/home/index.txt
create mode 100644 litellm/proxy/_experimental/out/lens/__next.!KGRhc2hib2FyZCk.lens.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/lens/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/lens/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/lens/index.html
create mode 100644 litellm/proxy/_experimental/out/lens/index.txt
create mode 100644 litellm/proxy/_experimental/out/model-insights/__next.!KGRhc2hib2FyZCk.model-insights.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/model-insights/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/model-insights/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/model-insights/index.html
create mode 100644 litellm/proxy/_experimental/out/model-insights/index.txt
create mode 100644 litellm/proxy/_experimental/out/moyai/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/moyai/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/moyai/__next.moyai.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/moyai/index.html
create mode 100644 litellm/proxy/_experimental/out/moyai/index.txt
create mode 100644 litellm/proxy/_experimental/out/telemetry/__next.!KGRhc2hib2FyZCk.telemetry.__PAGE__.txt
create mode 100644 litellm/proxy/_experimental/out/telemetry/__next._full.txt
create mode 100644 litellm/proxy/_experimental/out/telemetry/__next._tree.txt
create mode 100644 litellm/proxy/_experimental/out/telemetry/index.html
create mode 100644 litellm/proxy/_experimental/out/telemetry/index.txt
diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html
index c7297291b65..9e4050e8dcc 100644
--- a/litellm/proxy/_experimental/out/404.html
+++ b/litellm/proxy/_experimental/out/404.html
@@ -1 +1 @@
-LiteLLM Dashboard404: This page could not be found.
404
This page could not be found.
\ No newline at end of file
+LiteLLM Dashboard404: This page could not be found.
404
This page could not be found.
\ No newline at end of file
diff --git a/litellm/proxy/_experimental/out/404/index.html b/litellm/proxy/_experimental/out/404/index.html
index c7297291b65..9e4050e8dcc 100644
--- a/litellm/proxy/_experimental/out/404/index.html
+++ b/litellm/proxy/_experimental/out/404/index.html
@@ -1 +1 @@
-LiteLLM Dashboard404: This page could not be found.
404
This page could not be found.
\ No newline at end of file
+LiteLLM Dashboard404: This page could not be found.