scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

fix(rate-limits): stop the v3 limiter from creating provider-facing metadata on responses routes

Browse source
This commit is contained in:
Devin AI 2026-07-30 05:11:40 +00:00
parent 4d54324515
commit 68de15b01d
2 changed files with 102 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -28,6 +28,7 @@ from litellm import DualCache
from litellm._logging import verbose_proxy_logger
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
@ -2447,13 +2448,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
data["litellm_proxy_rate_limit_response"] = response
# Mirror into metadata so streaming success logging can find
# it via ``kwargs["litellm_params"]["metadata"]``.
self._stash_value_in_metadata_channels(
self._stash_value_in_internal_metadata(
data=data,
key=RATE_LIMIT_RESPONSE_KEY,
value=response,
)
if parallel_slot_id is not None:
self._stash_value_in_metadata_channels(
self._stash_value_in_internal_metadata(
data=data,
key=MAX_PARALLEL_SLOT_ACQUIRED_KEY,
value={
@ -2533,7 +2534,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
requested_model=requested_model,
)
else:
self._stash_value_in_metadata_channels(
self._stash_value_in_internal_metadata(
data=data,
key=RATE_LIMIT_DESCRIPTORS_KEY,
value=descriptors,
@ -2566,7 +2567,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
data["litellm_proxy_rate_limit_response"] = tpm_response
# Keep the metadata stash in sync when this is the
# first snapshot written.
self._stash_value_in_metadata_channels(
self._stash_value_in_internal_metadata(
data=data,
key=RATE_LIMIT_RESPONSE_KEY,
value=tpm_response,
@ -2803,19 +2804,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return merged
@staticmethod
def _stash_value_in_metadata_channels(
def _stash_value_in_internal_metadata(
data: Dict[str, Any],
key: str,
value: Any,
) -> None:
for channel in ("metadata", "litellm_metadata"):
existing = data.get(channel)
if isinstance(existing, dict):
existing[key] = value
elif channel == "metadata":
# ``litellm_metadata`` is owned by the router; don't conjure
# it here.
data[channel] = {key: value}
# Writes only the proxy-internal bucket. Routes that own
# ``litellm_metadata`` (Responses, /v1/messages, batches, files) expose
# ``metadata`` as a provider request parameter, so creating or adding to
# it here would forward internal state upstream.
_, metadata_bucket = get_or_create_metadata_bucket(data)
metadata_bucket[key] = value
@classmethod
def _stash_reservation_in_data(
@ -2831,11 +2830,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
"""
scopes_payload: Optional[List[List[str]]] = [[k, v] for k, v in reserved_scopes] if reserved_scopes else None
cls._stash_value_in_metadata_channels(data=data, key=TPM_RESERVED_TOKENS_KEY, value=estimated_tokens)
cls._stash_value_in_internal_metadata(data=data, key=TPM_RESERVED_TOKENS_KEY, value=estimated_tokens)
if reserved_model:
cls._stash_value_in_metadata_channels(data=data, key=TPM_RESERVED_MODEL_KEY, value=reserved_model)
cls._stash_value_in_internal_metadata(data=data, key=TPM_RESERVED_MODEL_KEY, value=reserved_model)
if scopes_payload is not None:
cls._stash_value_in_metadata_channels(data=data, key=TPM_RESERVED_SCOPES_KEY, value=scopes_payload)
cls._stash_value_in_internal_metadata(data=data, key=TPM_RESERVED_SCOPES_KEY, value=scopes_payload)
@staticmethod
def _lookup_stashed_value(
@ -2858,9 +2857,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return candidate
litellm_params = kwargs.get("litellm_params")
if isinstance(litellm_params, dict):
lp_metadata = litellm_params.get("metadata")
if isinstance(lp_metadata, dict):
candidate = lp_metadata.get(key)
for channel in ("litellm_metadata", "metadata"):
lp_metadata = litellm_params.get(channel)
if isinstance(lp_metadata, dict) and lp_metadata.get(key) is not None:
return lp_metadata[key]
if candidate is None and isinstance(standard_logging_metadata, dict):
candidate = standard_logging_metadata.get(key)
return candidate

83
tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
View file

@ -3165,6 +3165,89 @@ async def test_pre_call_hook_does_not_leak_internal_stash_to_request_body():
assert isinstance(metadata.get(RATE_LIMIT_DESCRIPTORS_KEY), list)
@pytest.mark.asyncio
@pytest.mark.parametrize("caller_metadata", [None, {"user_tag": "abc"}])
async def test_pre_call_hook_does_not_touch_provider_metadata_on_litellm_metadata_routes(
caller_metadata,
):
"""Regression for #35197: routes that own ``litellm_metadata`` (Responses,
/v1/messages, batches, files) send ``metadata`` to the provider, so the
limiter must never create it or write stash keys into it."""
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
_LITELLM_STASH_KEYS,
RATE_LIMIT_DESCRIPTORS_KEY,
RATE_LIMIT_RESPONSE_KEY,
TPM_RESERVED_TOKENS_KEY,
)
user_api_key_dict = UserAPIKeyAuth(
api_key=hash_token("sk-responses-metadata"),
tpm_limit=1000,
rpm_limit=5,
)
local_cache = DualCache()
handler = _PROXY_MaxParallelRequestsHandler(
internal_usage_cache=InternalUsageCache(local_cache),
)
async def mock_should_rate_limit(descriptors, **kwargs):
return {
"overall_code": "OK",
"statuses": [
{
"code": "OK",
"current_limit": 5,
"limit_remaining": 4,
"descriptor_key": d["key"],
"descriptor_value": d["value"],
"rate_limit_type": "requests",
}
for d in descriptors
],
}
async def mock_reserve_tpm_tokens(descriptors, estimated_tokens, **kwargs):
return {"overall_code": "OK", "statuses": []}
handler.should_rate_limit = mock_should_rate_limit
handler.reserve_tpm_tokens = mock_reserve_tpm_tokens
data: Dict[str, Any] = {
"model": "responses-model",
"input": "hello",
"litellm_metadata": {},
}
if caller_metadata is not None:
data["metadata"] = dict(caller_metadata)
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=local_cache,
data=data,
call_type="aresponses",
)
if caller_metadata is None:
assert "metadata" not in data, f"limiter created provider metadata: {data.get('metadata')!r}"
else:
assert data["metadata"] == caller_metadata
litellm_metadata = data["litellm_metadata"]
assert litellm_metadata.get(TPM_RESERVED_TOKENS_KEY)
assert isinstance(litellm_metadata.get(RATE_LIMIT_DESCRIPTORS_KEY), list)
assert litellm_metadata.get(RATE_LIMIT_RESPONSE_KEY)
leaked = [k for k in _LITELLM_STASH_KEYS if k in data]
assert not leaked, f"stash keys leaked to top level: {leaked}"
for key in _LITELLM_STASH_KEYS:
assert handler._lookup_stashed_value(
kwargs={"litellm_params": {"metadata": litellm_metadata}},
standard_logging_metadata=None,
key=key,
) == litellm_metadata.get(key)
@pytest.mark.asyncio
async def test_pre_call_hook_rejects_caller_supplied_stash_values():
"""Caller cannot pre-populate stash keys in body metadata to drive a