From 68de15b01d2c3fb43828651727d2c88af3b58bd0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 05:11:40 +0000 Subject: [PATCH] fix(rate-limits): stop the v3 limiter from creating provider-facing metadata on responses routes --- .../hooks/parallel_request_limiter_v3.py | 38 ++++----- .../hooks/test_parallel_request_limiter_v3.py | 83 +++++++++++++++++++ 2 files changed, 102 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index f72492881b3..719496785dd 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -28,6 +28,7 @@ from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) @@ -2447,13 +2448,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): data["litellm_proxy_rate_limit_response"] = response # Mirror into metadata so streaming success logging can find # it via ``kwargs["litellm_params"]["metadata"]``. - self._stash_value_in_metadata_channels( + self._stash_value_in_internal_metadata( data=data, key=RATE_LIMIT_RESPONSE_KEY, value=response, ) if parallel_slot_id is not None: - self._stash_value_in_metadata_channels( + self._stash_value_in_internal_metadata( data=data, key=MAX_PARALLEL_SLOT_ACQUIRED_KEY, value={ @@ -2533,7 +2534,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model=requested_model, ) else: - self._stash_value_in_metadata_channels( + self._stash_value_in_internal_metadata( data=data, key=RATE_LIMIT_DESCRIPTORS_KEY, value=descriptors, @@ -2566,7 +2567,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): data["litellm_proxy_rate_limit_response"] = tpm_response # Keep the metadata stash in sync when this is the # first snapshot written. - self._stash_value_in_metadata_channels( + self._stash_value_in_internal_metadata( data=data, key=RATE_LIMIT_RESPONSE_KEY, value=tpm_response, @@ -2803,19 +2804,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return merged @staticmethod - def _stash_value_in_metadata_channels( + def _stash_value_in_internal_metadata( data: Dict[str, Any], key: str, value: Any, ) -> None: - for channel in ("metadata", "litellm_metadata"): - existing = data.get(channel) - if isinstance(existing, dict): - existing[key] = value - elif channel == "metadata": - # ``litellm_metadata`` is owned by the router; don't conjure - # it here. - data[channel] = {key: value} + # Writes only the proxy-internal bucket. Routes that own + # ``litellm_metadata`` (Responses, /v1/messages, batches, files) expose + # ``metadata`` as a provider request parameter, so creating or adding to + # it here would forward internal state upstream. + _, metadata_bucket = get_or_create_metadata_bucket(data) + metadata_bucket[key] = value @classmethod def _stash_reservation_in_data( @@ -2831,11 +2830,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): """ scopes_payload: Optional[List[List[str]]] = [[k, v] for k, v in reserved_scopes] if reserved_scopes else None - cls._stash_value_in_metadata_channels(data=data, key=TPM_RESERVED_TOKENS_KEY, value=estimated_tokens) + cls._stash_value_in_internal_metadata(data=data, key=TPM_RESERVED_TOKENS_KEY, value=estimated_tokens) if reserved_model: - cls._stash_value_in_metadata_channels(data=data, key=TPM_RESERVED_MODEL_KEY, value=reserved_model) + cls._stash_value_in_internal_metadata(data=data, key=TPM_RESERVED_MODEL_KEY, value=reserved_model) if scopes_payload is not None: - cls._stash_value_in_metadata_channels(data=data, key=TPM_RESERVED_SCOPES_KEY, value=scopes_payload) + cls._stash_value_in_internal_metadata(data=data, key=TPM_RESERVED_SCOPES_KEY, value=scopes_payload) @staticmethod def _lookup_stashed_value( @@ -2858,9 +2857,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return candidate litellm_params = kwargs.get("litellm_params") if isinstance(litellm_params, dict): - lp_metadata = litellm_params.get("metadata") - if isinstance(lp_metadata, dict): - candidate = lp_metadata.get(key) + for channel in ("litellm_metadata", "metadata"): + lp_metadata = litellm_params.get(channel) + if isinstance(lp_metadata, dict) and lp_metadata.get(key) is not None: + return lp_metadata[key] if candidate is None and isinstance(standard_logging_metadata, dict): candidate = standard_logging_metadata.get(key) return candidate diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 9337050b61c..ee4eaa4decd 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3165,6 +3165,89 @@ async def test_pre_call_hook_does_not_leak_internal_stash_to_request_body(): assert isinstance(metadata.get(RATE_LIMIT_DESCRIPTORS_KEY), list) +@pytest.mark.asyncio +@pytest.mark.parametrize("caller_metadata", [None, {"user_tag": "abc"}]) +async def test_pre_call_hook_does_not_touch_provider_metadata_on_litellm_metadata_routes( + caller_metadata, +): + """Regression for #35197: routes that own ``litellm_metadata`` (Responses, + /v1/messages, batches, files) send ``metadata`` to the provider, so the + limiter must never create it or write stash keys into it.""" + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _LITELLM_STASH_KEYS, + RATE_LIMIT_DESCRIPTORS_KEY, + RATE_LIMIT_RESPONSE_KEY, + TPM_RESERVED_TOKENS_KEY, + ) + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-responses-metadata"), + tpm_limit=1000, + rpm_limit=5, + ) + local_cache = DualCache() + handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache), + ) + + async def mock_should_rate_limit(descriptors, **kwargs): + return { + "overall_code": "OK", + "statuses": [ + { + "code": "OK", + "current_limit": 5, + "limit_remaining": 4, + "descriptor_key": d["key"], + "descriptor_value": d["value"], + "rate_limit_type": "requests", + } + for d in descriptors + ], + } + + async def mock_reserve_tpm_tokens(descriptors, estimated_tokens, **kwargs): + return {"overall_code": "OK", "statuses": []} + + handler.should_rate_limit = mock_should_rate_limit + handler.reserve_tpm_tokens = mock_reserve_tpm_tokens + + data: Dict[str, Any] = { + "model": "responses-model", + "input": "hello", + "litellm_metadata": {}, + } + if caller_metadata is not None: + data["metadata"] = dict(caller_metadata) + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=data, + call_type="aresponses", + ) + + if caller_metadata is None: + assert "metadata" not in data, f"limiter created provider metadata: {data.get('metadata')!r}" + else: + assert data["metadata"] == caller_metadata + + litellm_metadata = data["litellm_metadata"] + assert litellm_metadata.get(TPM_RESERVED_TOKENS_KEY) + assert isinstance(litellm_metadata.get(RATE_LIMIT_DESCRIPTORS_KEY), list) + assert litellm_metadata.get(RATE_LIMIT_RESPONSE_KEY) + + leaked = [k for k in _LITELLM_STASH_KEYS if k in data] + assert not leaked, f"stash keys leaked to top level: {leaked}" + + for key in _LITELLM_STASH_KEYS: + assert handler._lookup_stashed_value( + kwargs={"litellm_params": {"metadata": litellm_metadata}}, + standard_logging_metadata=None, + key=key, + ) == litellm_metadata.get(key) + + @pytest.mark.asyncio async def test_pre_call_hook_rejects_caller_supplied_stash_values(): """Caller cannot pre-populate stash keys in body metadata to drive a